<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package runtime

import (
	"math"
	"os"

	"github.com/islisp-dev/iris/runtime/env"
	"github.com/islisp-dev/iris/runtime/ilos"
	"github.com/islisp-dev/iris/runtime/ilos/class"
	"github.com/islisp-dev/iris/runtime/ilos/instance"
)

func TopLevelHander(e env.Environment, c ilos.Instance) (ilos.Instance, ilos.Instance) {
	return nil, c
}

var TopLevel = env.NewEnvironment(
	instance.NewStream(os.Stdin, nil),
	instance.NewStream(nil, os.Stdout),
	instance.NewStream(nil, os.Stderr),
	instance.NewFunction(instance.NewSymbol("TOP-LEVEL-HANDLER"), TopLevelHander),
)

func defclass(name string, class ilos.Class) {
	symbol := instance.NewSymbol(name)
	TopLevel.Class.Define(symbol, class)
}

func defspecial(name string, function interface{}) {
	symbol := instance.NewSymbol(name)
	TopLevel.Special.Define(symbol, instance.NewFunction(func2symbol(function), function))
}

func defun(name string, function interface{}) {
	symbol := instance.NewSymbol(name)
	TopLevel.Function.Define(symbol, instance.NewFunction(symbol, function))
}

func defgeneric(name string, function interface{}) {
	symbol := instance.NewSymbol(name)
	lambdaList, _ := List(TopLevel, instance.NewSymbol("FIRST"), instance.NewSymbol("&REST"), instance.NewSymbol("REST"))
	generic := instance.NewGenericFunction(symbol, lambdaList, T, class.GenericFunction)
	generic.(*instance.GenericFunction).AddMethod(nil, lambdaList, []ilos.Class{class.StandardClass}, instance.NewFunction(symbol, function))
	TopLevel.Function.Define(symbol, generic)
}

func defglobal(name string, value ilos.Instance) {
	symbol := instance.NewSymbol(name)
	TopLevel.Variable.Define(symbol, value)
}

func init() {
	defglobal("*PI*", instance.Float(math.Pi))
	defglobal("*MOST-POSITIVE-FLOAT*", MostPositiveFloat)
	defglobal("*MOST-NEGATIVE-FLOAT*", MostNegativeFloat)
	defun("-", Substruct)
	defun("+", Add)
	defun("*", Multiply)
	defun("<", NumberLessThan)
	defun("<=", NumberLessThanOrEqual)
	defun("=", NumberEqual)
	defun(">", NumberGreaterThan)
	defun(">=", NumberGreaterThanOrEqual)
	defspecial("QUASIQUOTE", Quasiquote)
	defun("ABS", Abs)
	defspecial("AND", And)
	defun("APPEND", Append)
	defun("APPLY", Apply)
	defun("ARRAY-DIMENSIONS", ArrayDimensions)
	defun("AREF", Aref)
	defun("ASSOC", Assoc)
	// TODO: defspecial2("ASSURE", Assure)
	defun("ATAN", Atan)
	defun("ATAN2", Atan2)
	defun("ATANH", Atanh)
	defun("BASIC-ARRAY*-P", BasicArrayStarP)
	defun("BASIC-ARRAY-P", BasicArrayP)
	defun("BASIC-VECTOR-P", BasicVectorP)
	defspecial("BLOCK", Block)
	defun("CAR", Car)
	defspecial("CASE", Case)
	defspecial("CASE-USING", CaseUsing)
	defspecial("CATCH", Catch)
	defun("CDR", Cdr)
	defun("CEILING", Ceiling)
	defun("CERROR", Cerror)
	defun("CHAR-INDEX", CharIndex)
	defun("CHAR/=", CharNotEqual)
	defun("CHAR<", CharLessThan)
	defun("CHAR<=", CharLessThanOrEqual)
	defun("CHAR=", CharEqual)
	defun("CHAR>", CharGreaterThan)
	defun("CHAR>=", CharGreaterThanOrEqual)
	defun("CHARACTERP", Characterp)
	defspecial("CLASS", Class)
	defun("CLASS-OF", ClassOf)
	defun("CLOSE", Close)
	// TODO defun2("COERCION", Coercion)
	defspecial("COND", Cond)
	defun("CONDITION-CONTINUABLE", ConditionContinuable)
	defun("CONS", Cons)
	defun("CONSP", Consp)
	defun("CONTINUE-CONDITION", ContinueCondition)
	defspecial("CONVERT", Convert)
	defun("COS", Cos)
	defun("COSH", Cosh)
	defgeneric("CREATE", Create) //TODO Change to generic function
	defun("CREATE-ARRAY", CreateArray)
	defun("CREATE-LIST", CreateList)
	defun("CREATE-STRING", CreateString)
	defun("CREATE-STRING-INPUT-STREAM", CreateStringInputStream)
	defun("CREATE-STRING-OUTPUT-STREAM", CreateStringOutputStream)
	defun("CREATE-VECTOR", CreateVector)
	defspecial("DEFCLASS", Defclass)
	defspecial("DEFCONSTANT", Defconstant)
	defspecial("DEFDYNAMIC", Defdynamic)
	defspecial("DEFGENERIC", Defgeneric)
	defspecial("DEFMETHOD", Defmethod)
	defspecial("DEFGLOBAL", Defglobal)
	defspecial("DEFMACRO", Defmacro)
	defspecial("DEFUN", Defun)
	defun("DIV", Div)
	defspecial("DYNAMIC", Dynamic)
	defspecial("DYNAMIC-LET", DynamicLet)
	defun("ELT", Elt)
	defun("EQ", Eq)
	defun("EQL", Eql)
	defun("EQUAL", Equal)
	defun("ERROR", Error)
	defun("ERROR-OUTPUT", ErrorOutput)
	defun("EXP", Exp)
	defun("EXPT", Expt)
	// TODO defun2("FILE-LENGTH", FileLength)
	// TODO defun2("FILE-POSITION", FilePosition)
	// TODO defun2("FINISH-OUTPUT", FinishOutput)
	defspecial("FLET", Flet)
	defun("FLOAT", Float)
	defun("FLOATP", Floatp)
	defun("FLOOR", Floor)
	defspecial("FOR", For)
	defun("FORMAT", Format)
	defun("FORMAT-CHAR", FormatChar)
	defun("FORMAT-FLOAT", FormatFloat)
	defun("FORMAT-FRESH-LINE", FormatFreshLine)
	defun("FORMAT-INTEGER", FormatInteger)
	defun("FORMAT-OBJECT", FormatObject)
	defun("FORMAT-TAB", FormatTab)
	defun("FUNCALL", Funcall)
	defspecial("FUNCTION", Function)
	defun("FUNCTIONP", Functionp)
	defun("GAREF", Garef)
	defun("GCD", Gcd)
	defun("GENERAL-ARRAY*-P", GeneralArrayStarP)
	defun("GENERAL-VECTOR-P", GeneralVectorP)
	// TODO defun2("GENERIC-FUNCTION-P", GenericFunctionP)
	defun("GENSYM", Gensym)
	// TODO defun2("GET-INTERNAL-REAL-TIME", GetInternalRealTime)
	// TODO defun2("GET-INTERNAL-RUN-TIME", GetInternalRunTime)
	defun("GET-OUTPUT-STREAM-STRING", GetOutputStreamString)
	// TODO defun2("GET-UNIVERSAL-TIME", GetUniversalTime)
	defspecial("GO", Go)
	defun("IDENTITY", Identity)
	defspecial("IF", If)
	// TODO defspecial2("IGNORE-ERRORS", IgnoreErrors)
	defgeneric("INITIALIZE-OBJECT", InitializeObject) // TODO change generic function
	defun("INPUT-STREAM-P", InputStreamP)
	defun("INSTANCEP", Instancep)
	// TODO defun2("INTEGER", Integer)
	defun("INTEGERP", Integerp)
	// TODO defun2("INTERNAL-TIME-UNITS-PER-SECOND", InternalTimeUnitsPerSecond)
	defun("ISQRT", Isqrt)
	defspecial("LABELS", Labels)
	defspecial("LAMBDA", Lambda)
	defun("LCM", Lcm)
	defun("LENGTH", Length)
	defspecial("LET", Let)
	defspecial("LET*", LetStar)
	defun("LIST", List)
	defun("LISTP", Listp)
	defun("LOG", Log)
	defun("MAP-INTO", MapInto)
	defun("MAPC", Mapc)
	defun("MAPCAN", Mapcan)
	defun("MAPCAR", Mapcar)
	defun("MAPCON", Mapcon)
	defun("MAPL", Mapl)
	defun("MAPLIST", Maplist)
	defun("MAX", Max)
	defun("MEMBER", Member)
	defun("MIN", Min)
	defun("MOD", Mod)
	defglobal("NI-L", Nil)
	defun("NOT", Not)
	defun("NREVERSE", Nreverse)
	defun("NULL", Null)
	defun("NUMBERP", Numberp)
	defun("OPEN-INPUT-FILE", OpenInputFile)
	defun("OPEN-IO-FILE", OpenIoFile)
	defun("OPEN-OUTPUT-FILE", OpenOutputFile)
	defun("OPEN-STREAM-P", OpenStreamP)
	defspecial("OR", Or)
	defun("OUTPUT-STREAM-P", OutputStreamP)
	defun("PARSE-NUMBER", ParseNumber)
	// TODO defun2("PREVIEW-CHAR", PreviewChar)
	// TODO defun2("PROVE-FILE", ProveFile)
	defspecial("PROGN", Progn)
	defun("PROPERTY", Property)
	defspecial("QUASIQUOTE", Quasiquote)
	defspecial("QUOTE", Quote)
	defun("QUOTIENT", Quotient)
	defun("READ", Read)
	// TODO defun2("READ-BYTE", ReadByte)
	defun("READ-CHAR", ReadChar)
	defun("READ-LINE", ReadLine)
	defun("REMOVE-PROPERTY", RemoveProperty)
	defun("REPORT-CONDITION", ReportCondition)
	defspecial("RETURN-FROM", ReturnFrom)
	defun("REVERSE", Reverse)
	defun("ROUND", Round)
	defun("SET-AREF", SetAref)
	defun("(SETF AREF)", SetAref)
	defun("SET-CAR", SetCar)
	defun("(SETF CAR)", SetCar)
	defun("SET-CDR", SetCdr)
	defun("(SETF CDR)", SetCdr)
	defun("SET-DYNAMIC", SetDynamic)
	defun("(SETF DYNAMIC)", SetDynamic)
	defun("SET-ELT", SetElt)
	defun("(SETF ELT)", SetElt)
	// TODO defun2("SET-FILE-POSITION", SetFilePosition)
	defun("SET-GAREF", SetGaref)
	defun("(SETF GAREF)", SetGaref)
	defun("SET-PROPERTY", SetProperty)
	defun("(SETF PROPERTY)", SetProperty)
	defspecial("SETF", Setf)
	defspecial("SETQ", Setq)
	defun("SIGNAL-CONDITION", SignalCondition)
	// TODO defun2("SIMPLE-ERROR-FORMAT-ARGUMENTS", SimpleErrorFormatArguments)
	// TODO defun2("SIMPLE-ERROR-FORMAT-STRING", SimpleErrorFormatString)
	defun("SIN", Sin)
	defun("SINH", Sinh)
	defun("SQRT", Sqrt)
	defun("STANDARD-INPUT", StandardInput)
	defun("STANDARD-OUTPUT", StandardOutput)
	defun("STREAM-READY-P", StreamReadyP)
	defun("STREAMP", Streamp)
	defun("STRING-APPEND", StringAppend)
	defun("STRING-INDEX", StringIndex)
	defun("STRING/=", StringNotEqual)
	defun("STRING>", StringGreaterThan)
	defun("STRING>=", StringGreaterThanOrEqual)
	defun("STRING=", StringEqual)
	defun("STRING<", StringLessThan)
	defun("STRING<=", StringLessThanOrEqual)
	defun("STRINGP", Stringp)
	defun("SUBCLASSP", Subclassp)
	defun("SUBSEQ", Subseq)
	defun("SYMBOLP", Symbolp)
	defglobal("T", T)
	defspecial("TAGBODY", Tagbody)
	defspecial("TAN", Tan)
	defspecial("TANH", Tanh)
	// TODO defspecial2("THE", The)
	defspecial("THROW", Throw)
	defun("TRUNCATE", Truncate)
	// TODO defun1("UNDEFINED-ENTITY-NAME", UndefinedEntityName)
	// TODO defun2("UNDEFINED-ENTITY-NAMESPACE", UndefinedEntityNamespace)
	defspecial("UNWIND-PROTECT", UnwindProtect)
	defun("VECTOR", Vector)
	defspecial("WHILE", While)
	defspecial("WITH-ERROR-OUTPUT", WithErrorOutput)
	defspecial("WITH-HANDLER", WithHandler)
	defspecial("WITH-OPEN-INPUT-FILE", WithOpenInputFile)
	defspecial("WITH-OPEN-OUTPUT-FILE", WithOpenOutputFile)
	defspecial("WITH-STANDARD-INPUT", WithStandardInput)
	defspecial("WITH-STANDARD-OUTPUT", WithStandardOutput)
	// TODO defun2("WRITE-BYTE", WriteByte)

	defclass("<OBJECT>", class.Object)
	defclass("<BUILT-IN-CLASS>", class.BuiltInClass)
	defclass("<STANDARD-CLASS>", class.StandardClass)
	defclass("<BASIC-ARRAY>", class.BasicArray)
	defclass("<BASIC-ARRAY-STAR>", class.BasicArrayStar)
	defclass("<GENERAL-ARRAY-STAR>", class.GeneralArrayStar)
	defclass("<BASIC-VECTOR>", class.BasicVector)
	defclass("<GENERAL-VECTOR>", class.GeneralVector)
	defclass("<STRING>", class.String)
	defclass("<CHARACTER>", class.Character)
	defclass("<FUNCTION>", class.Function)
	defclass("<GENERIC-FUNCTION>", class.GenericFunction)
	defclass("<STANDARD-GENERIC-FUNCTION>", class.StandardGenericFunction)
	defclass("<LIST>", class.List)
	defclass("<CONS>", class.Cons)
	defclass("<NULL>", class.Null)
	defclass("<SYMBOL>", class.Symbol)
	defclass("<NUMBER>", class.Number)
	defclass("<INTEGER>", class.Integer)
	defclass("<FLOAT>", class.Float)
	defclass("<SERIOUS-CONDITION>", class.SeriousCondition)
	defclass("<ERROR>", class.Error)
	defclass("<ARITHMETIC-ERROR>", class.ArithmeticError)
	defclass("<DIVISION-BY-ZERO>", class.DivisionByZero)
	defclass("<FLOATING-POINT-ONDERFLOW>", class.FloatingPointOnderflow)
	defclass("<FLOATING-POINT-UNDERFLOW>", class.FloatingPointUnderflow)
	defclass("<CONTROL-ERROR>", class.ControlError)
	defclass("<PARSE-ERROR>", class.ParseError)
	defclass("<PROGRAM-ERROR>", class.ProgramError)
	defclass("<DOMAIN-ERROR>", class.DomainError)
	defclass("<UNDEFINED-ENTITY>", class.UndefinedEntity)
	defclass("<UNDEFINED-VARIABLE>", class.UndefinedVariable)
	defclass("<UNDEFINED-FUNCTION>", class.UndefinedFunction)
	defclass("<SIMPLE-ERROR>", class.SimpleError)
	defclass("<STREAM-ERROR>", class.StreamError)
	defclass("<END-OF-STREAM>", class.EndOfStream)
	defclass("<STORAGE-EXHAUSTED>", class.StorageExhausted)
	defclass("<STANDARD-OBJECT>", class.StandardObject)
	defclass("<STREAM>", class.Stream)
}
<commit_msg>Add PreviewChar and ProbeFile<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package runtime

import (
	"math"
	"os"

	"github.com/islisp-dev/iris/runtime/env"
	"github.com/islisp-dev/iris/runtime/ilos"
	"github.com/islisp-dev/iris/runtime/ilos/class"
	"github.com/islisp-dev/iris/runtime/ilos/instance"
)

func TopLevelHander(e env.Environment, c ilos.Instance) (ilos.Instance, ilos.Instance) {
	return nil, c
}

var TopLevel = env.NewEnvironment(
	instance.NewStream(os.Stdin, nil),
	instance.NewStream(nil, os.Stdout),
	instance.NewStream(nil, os.Stderr),
	instance.NewFunction(instance.NewSymbol("TOP-LEVEL-HANDLER"), TopLevelHander),
)

func defclass(name string, class ilos.Class) {
	symbol := instance.NewSymbol(name)
	TopLevel.Class.Define(symbol, class)
}

func defspecial(name string, function interface{}) {
	symbol := instance.NewSymbol(name)
	TopLevel.Special.Define(symbol, instance.NewFunction(func2symbol(function), function))
}

func defun(name string, function interface{}) {
	symbol := instance.NewSymbol(name)
	TopLevel.Function.Define(symbol, instance.NewFunction(symbol, function))
}

func defgeneric(name string, function interface{}) {
	symbol := instance.NewSymbol(name)
	lambdaList, _ := List(TopLevel, instance.NewSymbol("FIRST"), instance.NewSymbol("&REST"), instance.NewSymbol("REST"))
	generic := instance.NewGenericFunction(symbol, lambdaList, T, class.GenericFunction)
	generic.(*instance.GenericFunction).AddMethod(nil, lambdaList, []ilos.Class{class.StandardClass}, instance.NewFunction(symbol, function))
	TopLevel.Function.Define(symbol, generic)
}

func defglobal(name string, value ilos.Instance) {
	symbol := instance.NewSymbol(name)
	TopLevel.Variable.Define(symbol, value)
}

func init() {
	defglobal("*PI*", instance.Float(math.Pi))
	defglobal("*MOST-POSITIVE-FLOAT*", MostPositiveFloat)
	defglobal("*MOST-NEGATIVE-FLOAT*", MostNegativeFloat)
	defun("-", Substruct)
	defun("+", Add)
	defun("*", Multiply)
	defun("<", NumberLessThan)
	defun("<=", NumberLessThanOrEqual)
	defun("=", NumberEqual)
	defun(">", NumberGreaterThan)
	defun(">=", NumberGreaterThanOrEqual)
	defspecial("QUASIQUOTE", Quasiquote)
	defun("ABS", Abs)
	defspecial("AND", And)
	defun("APPEND", Append)
	defun("APPLY", Apply)
	defun("ARRAY-DIMENSIONS", ArrayDimensions)
	defun("AREF", Aref)
	defun("ASSOC", Assoc)
	// TODO: defspecial2("ASSURE", Assure)
	defun("ATAN", Atan)
	defun("ATAN2", Atan2)
	defun("ATANH", Atanh)
	defun("BASIC-ARRAY*-P", BasicArrayStarP)
	defun("BASIC-ARRAY-P", BasicArrayP)
	defun("BASIC-VECTOR-P", BasicVectorP)
	defspecial("BLOCK", Block)
	defun("CAR", Car)
	defspecial("CASE", Case)
	defspecial("CASE-USING", CaseUsing)
	defspecial("CATCH", Catch)
	defun("CDR", Cdr)
	defun("CEILING", Ceiling)
	defun("CERROR", Cerror)
	defun("CHAR-INDEX", CharIndex)
	defun("CHAR/=", CharNotEqual)
	defun("CHAR<", CharLessThan)
	defun("CHAR<=", CharLessThanOrEqual)
	defun("CHAR=", CharEqual)
	defun("CHAR>", CharGreaterThan)
	defun("CHAR>=", CharGreaterThanOrEqual)
	defun("CHARACTERP", Characterp)
	defspecial("CLASS", Class)
	defun("CLASS-OF", ClassOf)
	defun("CLOSE", Close)
	// SKIP defun2("COERCION", Coercion)
	defspecial("COND", Cond)
	defun("CONDITION-CONTINUABLE", ConditionContinuable)
	defun("CONS", Cons)
	defun("CONSP", Consp)
	defun("CONTINUE-CONDITION", ContinueCondition)
	defspecial("CONVERT", Convert)
	defun("COS", Cos)
	defun("COSH", Cosh)
	defgeneric("CREATE", Create) //TODO Change to generic function
	defun("CREATE-ARRAY", CreateArray)
	defun("CREATE-LIST", CreateList)
	defun("CREATE-STRING", CreateString)
	defun("CREATE-STRING-INPUT-STREAM", CreateStringInputStream)
	defun("CREATE-STRING-OUTPUT-STREAM", CreateStringOutputStream)
	defun("CREATE-VECTOR", CreateVector)
	defspecial("DEFCLASS", Defclass)
	defspecial("DEFCONSTANT", Defconstant)
	defspecial("DEFDYNAMIC", Defdynamic)
	defspecial("DEFGENERIC", Defgeneric)
	defspecial("DEFMETHOD", Defmethod)
	defspecial("DEFGLOBAL", Defglobal)
	defspecial("DEFMACRO", Defmacro)
	defspecial("DEFUN", Defun)
	defun("DIV", Div)
	defspecial("DYNAMIC", Dynamic)
	defspecial("DYNAMIC-LET", DynamicLet)
	defun("ELT", Elt)
	defun("EQ", Eq)
	defun("EQL", Eql)
	defun("EQUAL", Equal)
	defun("ERROR", Error)
	defun("ERROR-OUTPUT", ErrorOutput)
	defun("EXP", Exp)
	defun("EXPT", Expt)
	// TODO defun2("FILE-LENGTH", FileLength)
	// TODO defun2("FILE-POSITION", FilePosition)
	// TODO defun2("FINISH-OUTPUT", FinishOutput)
	defspecial("FLET", Flet)
	defun("FLOAT", Float)
	defun("FLOATP", Floatp)
	defun("FLOOR", Floor)
	defspecial("FOR", For)
	defun("FORMAT", Format)
	defun("FORMAT-CHAR", FormatChar)
	defun("FORMAT-FLOAT", FormatFloat)
	defun("FORMAT-FRESH-LINE", FormatFreshLine)
	defun("FORMAT-INTEGER", FormatInteger)
	defun("FORMAT-OBJECT", FormatObject)
	defun("FORMAT-TAB", FormatTab)
	defun("FUNCALL", Funcall)
	defspecial("FUNCTION", Function)
	defun("FUNCTIONP", Functionp)
	defun("GAREF", Garef)
	defun("GCD", Gcd)
	defun("GENERAL-ARRAY*-P", GeneralArrayStarP)
	defun("GENERAL-VECTOR-P", GeneralVectorP)
	// TODO defun2("GENERIC-FUNCTION-P", GenericFunctionP)
	defun("GENSYM", Gensym)
	// TODO defun2("GET-INTERNAL-REAL-TIME", GetInternalRealTime)
	// TODO defun2("GET-INTERNAL-RUN-TIME", GetInternalRunTime)
	defun("GET-OUTPUT-STREAM-STRING", GetOutputStreamString)
	// TODO defun2("GET-UNIVERSAL-TIME", GetUniversalTime)
	defspecial("GO", Go)
	defun("IDENTITY", Identity)
	defspecial("IF", If)
	// TODO defspecial2("IGNORE-ERRORS", IgnoreErrors)
	defgeneric("INITIALIZE-OBJECT", InitializeObject) // TODO change generic function
	defun("INPUT-STREAM-P", InputStreamP)
	defun("INSTANCEP", Instancep)
	defun("INTEGERP", Integerp)
	// TODO defun2("INTERNAL-TIME-UNITS-PER-SECOND", InternalTimeUnitsPerSecond)
	defun("ISQRT", Isqrt)
	defspecial("LABELS", Labels)
	defspecial("LAMBDA", Lambda)
	defun("LCM", Lcm)
	defun("LENGTH", Length)
	defspecial("LET", Let)
	defspecial("LET*", LetStar)
	defun("LIST", List)
	defun("LISTP", Listp)
	defun("LOG", Log)
	defun("MAP-INTO", MapInto)
	defun("MAPC", Mapc)
	defun("MAPCAN", Mapcan)
	defun("MAPCAR", Mapcar)
	defun("MAPCON", Mapcon)
	defun("MAPL", Mapl)
	defun("MAPLIST", Maplist)
	defun("MAX", Max)
	defun("MEMBER", Member)
	defun("MIN", Min)
	defun("MOD", Mod)
	defglobal("NI-L", Nil)
	defun("NOT", Not)
	defun("NREVERSE", Nreverse)
	defun("NULL", Null)
	defun("NUMBERP", Numberp)
	defun("OPEN-INPUT-FILE", OpenInputFile)
	defun("OPEN-IO-FILE", OpenIoFile)
	defun("OPEN-OUTPUT-FILE", OpenOutputFile)
	defun("OPEN-STREAM-P", OpenStreamP)
	defspecial("OR", Or)
	defun("OUTPUT-STREAM-P", OutputStreamP)
	defun("PARSE-NUMBER", ParseNumber)
	defun("PREVIEW-CHAR", PreviewChar)
	defun("PROBE-FILE", ProbeFile)
	defspecial("PROGN", Progn)
	defun("PROPERTY", Property)
	defspecial("QUASIQUOTE", Quasiquote)
	defspecial("QUOTE", Quote)
	defun("QUOTIENT", Quotient)
	defun("READ", Read)
	// TODO defun2("READ-BYTE", ReadByte)
	defun("READ-CHAR", ReadChar)
	defun("READ-LINE", ReadLine)
	defun("REMOVE-PROPERTY", RemoveProperty)
	defun("REPORT-CONDITION", ReportCondition)
	defspecial("RETURN-FROM", ReturnFrom)
	defun("REVERSE", Reverse)
	defun("ROUND", Round)
	defun("SET-AREF", SetAref)
	defun("(SETF AREF)", SetAref)
	defun("SET-CAR", SetCar)
	defun("(SETF CAR)", SetCar)
	defun("SET-CDR", SetCdr)
	defun("(SETF CDR)", SetCdr)
	defun("SET-DYNAMIC", SetDynamic)
	defun("(SETF DYNAMIC)", SetDynamic)
	defun("SET-ELT", SetElt)
	defun("(SETF ELT)", SetElt)
	// TODO defun2("SET-FILE-POSITION", SetFilePosition)
	defun("SET-GAREF", SetGaref)
	defun("(SETF GAREF)", SetGaref)
	defun("SET-PROPERTY", SetProperty)
	defun("(SETF PROPERTY)", SetProperty)
	defspecial("SETF", Setf)
	defspecial("SETQ", Setq)
	defun("SIGNAL-CONDITION", SignalCondition)
	// TODO defun2("SIMPLE-ERROR-FORMAT-ARGUMENTS", SimpleErrorFormatArguments)
	// TODO defun2("SIMPLE-ERROR-FORMAT-STRING", SimpleErrorFormatString)
	defun("SIN", Sin)
	defun("SINH", Sinh)
	defun("SQRT", Sqrt)
	defun("STANDARD-INPUT", StandardInput)
	defun("STANDARD-OUTPUT", StandardOutput)
	defun("STREAM-READY-P", StreamReadyP)
	defun("STREAMP", Streamp)
	defun("STRING-APPEND", StringAppend)
	defun("STRING-INDEX", StringIndex)
	defun("STRING/=", StringNotEqual)
	defun("STRING>", StringGreaterThan)
	defun("STRING>=", StringGreaterThanOrEqual)
	defun("STRING=", StringEqual)
	defun("STRING<", StringLessThan)
	defun("STRING<=", StringLessThanOrEqual)
	defun("STRINGP", Stringp)
	defun("SUBCLASSP", Subclassp)
	defun("SUBSEQ", Subseq)
	defun("SYMBOLP", Symbolp)
	defglobal("T", T)
	defspecial("TAGBODY", Tagbody)
	defspecial("TAN", Tan)
	defspecial("TANH", Tanh)
	// TODO defspecial2("THE", The)
	defspecial("THROW", Throw)
	defun("TRUNCATE", Truncate)
	// TODO defun1("UNDEFINED-ENTITY-NAME", UndefinedEntityName)
	// TODO defun2("UNDEFINED-ENTITY-NAMESPACE", UndefinedEntityNamespace)
	defspecial("UNWIND-PROTECT", UnwindProtect)
	defun("VECTOR", Vector)
	defspecial("WHILE", While)
	defspecial("WITH-ERROR-OUTPUT", WithErrorOutput)
	defspecial("WITH-HANDLER", WithHandler)
	defspecial("WITH-OPEN-INPUT-FILE", WithOpenInputFile)
	defspecial("WITH-OPEN-OUTPUT-FILE", WithOpenOutputFile)
	defspecial("WITH-STANDARD-INPUT", WithStandardInput)
	defspecial("WITH-STANDARD-OUTPUT", WithStandardOutput)
	// TODO defun2("WRITE-BYTE", WriteByte)

	defclass("<OBJECT>", class.Object)
	defclass("<BUILT-IN-CLASS>", class.BuiltInClass)
	defclass("<STANDARD-CLASS>", class.StandardClass)
	defclass("<BASIC-ARRAY>", class.BasicArray)
	defclass("<BASIC-ARRAY-STAR>", class.BasicArrayStar)
	defclass("<GENERAL-ARRAY-STAR>", class.GeneralArrayStar)
	defclass("<BASIC-VECTOR>", class.BasicVector)
	defclass("<GENERAL-VECTOR>", class.GeneralVector)
	defclass("<STRING>", class.String)
	defclass("<CHARACTER>", class.Character)
	defclass("<FUNCTION>", class.Function)
	defclass("<GENERIC-FUNCTION>", class.GenericFunction)
	defclass("<STANDARD-GENERIC-FUNCTION>", class.StandardGenericFunction)
	defclass("<LIST>", class.List)
	defclass("<CONS>", class.Cons)
	defclass("<NULL>", class.Null)
	defclass("<SYMBOL>", class.Symbol)
	defclass("<NUMBER>", class.Number)
	defclass("<INTEGER>", class.Integer)
	defclass("<FLOAT>", class.Float)
	defclass("<SERIOUS-CONDITION>", class.SeriousCondition)
	defclass("<ERROR>", class.Error)
	defclass("<ARITHMETIC-ERROR>", class.ArithmeticError)
	defclass("<DIVISION-BY-ZERO>", class.DivisionByZero)
	defclass("<FLOATING-POINT-ONDERFLOW>", class.FloatingPointOnderflow)
	defclass("<FLOATING-POINT-UNDERFLOW>", class.FloatingPointUnderflow)
	defclass("<CONTROL-ERROR>", class.ControlError)
	defclass("<PARSE-ERROR>", class.ParseError)
	defclass("<PROGRAM-ERROR>", class.ProgramError)
	defclass("<DOMAIN-ERROR>", class.DomainError)
	defclass("<UNDEFINED-ENTITY>", class.UndefinedEntity)
	defclass("<UNDEFINED-VARIABLE>", class.UndefinedVariable)
	defclass("<UNDEFINED-FUNCTION>", class.UndefinedFunction)
	defclass("<SIMPLE-ERROR>", class.SimpleError)
	defclass("<STREAM-ERROR>", class.StreamError)
	defclass("<END-OF-STREAM>", class.EndOfStream)
	defclass("<STORAGE-EXHAUSTED>", class.StorageExhausted)
	defclass("<STANDARD-OBJECT>", class.StandardObject)
	defclass("<STREAM>", class.Stream)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package glusterfs

import (
	"math/rand"
	"os"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/volume"
)

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&glusterfsPlugin{nil}}
}

type glusterfsPlugin struct {
	host volume.VolumeHost
}

var _ volume.VolumePlugin = &glusterfsPlugin{}
var _ volume.PersistentVolumePlugin = &glusterfsPlugin{}

const (
	glusterfsPluginName = "kubernetes.io/glusterfs"
)

func (plugin *glusterfsPlugin) Init(host volume.VolumeHost) {
	plugin.host = host
}

func (plugin *glusterfsPlugin) Name() string {
	return glusterfsPluginName
}

func (plugin *glusterfsPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Glusterfs != nil) ||
		(spec.Volume != nil && spec.Volume.Glusterfs != nil)
}

func (plugin *glusterfsPlugin) GetAccessModes() []api.PersistentVolumeAccessMode {
	return []api.PersistentVolumeAccessMode{
		api.ReadWriteOnce,
		api.ReadOnlyMany,
		api.ReadWriteMany,
	}
}

func (plugin *glusterfsPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, _ volume.VolumeOptions, mounter mount.Interface) (volume.Builder, error) {
	source, _ := plugin.getGlusterVolumeSource(spec)
	ep_name := source.EndpointsName
	ns := pod.Namespace
	ep, err := plugin.host.GetKubeClient().Endpoints(ns).Get(ep_name)
	if err != nil {
		glog.Errorf("Glusterfs: failed to get endpoints %s[%v]", ep_name, err)
		return nil, err
	}
	glog.V(1).Infof("Glusterfs: endpoints %v", ep)
	return plugin.newBuilderInternal(spec, ep, pod, mounter, exec.New())
}

func (plugin *glusterfsPlugin) getGlusterVolumeSource(spec *volume.Spec) (*api.GlusterfsVolumeSource, bool) {
	// Glusterfs volumes used directly in a pod have a ReadOnly flag set by the pod author.
	// Glusterfs volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
	if spec.Volume != nil && spec.Volume.Glusterfs != nil {
		return spec.Volume.Glusterfs, spec.Volume.Glusterfs.ReadOnly
	} else {
		return spec.PersistentVolume.Spec.Glusterfs, spec.ReadOnly
	}
}

func (plugin *glusterfsPlugin) newBuilderInternal(spec *volume.Spec, ep *api.Endpoints, pod *api.Pod, mounter mount.Interface, exe exec.Interface) (volume.Builder, error) {
	source, readOnly := plugin.getGlusterVolumeSource(spec)
	return &glusterfsBuilder{
		glusterfs: &glusterfs{
			volName: spec.Name(),
			mounter: mounter,
			pod:     pod,
			plugin:  plugin,
		},
		hosts:    ep,
		path:     source.Path,
		readOnly: readOnly,
		exe:      exe}, nil
}

func (plugin *glusterfsPlugin) NewCleaner(volName string, podUID types.UID, mounter mount.Interface) (volume.Cleaner, error) {
	return plugin.newCleanerInternal(volName, podUID, mounter)
}

func (plugin *glusterfsPlugin) newCleanerInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Cleaner, error) {
	return &glusterfsCleaner{&glusterfs{
		volName: volName,
		mounter: mounter,
		pod:     &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
		plugin:  plugin,
	}}, nil
}

// Glusterfs volumes represent a bare host file or directory mount of an Glusterfs export.
type glusterfs struct {
	volName string
	pod     *api.Pod
	mounter mount.Interface
	plugin  *glusterfsPlugin
}

type glusterfsBuilder struct {
	*glusterfs
	hosts    *api.Endpoints
	path     string
	readOnly bool
	exe      exec.Interface
}

var _ volume.Builder = &glusterfsBuilder{}

// SetUp attaches the disk and bind mounts to the volume path.
func (b *glusterfsBuilder) SetUp() error {
	return b.SetUpAt(b.GetPath())
}

func (b *glusterfsBuilder) SetUpAt(dir string) error {
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	glog.V(4).Infof("Glusterfs: mount set up: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		return nil
	}

	os.MkdirAll(dir, 0750)
	err = b.setUpAtInternal(dir)
	if err == nil {
		return nil
	}

	// Cleanup upon failure.
	c := &glusterfsCleaner{b.glusterfs}
	c.cleanup(dir)
	return err
}

func (b *glusterfsBuilder) IsReadOnly() bool {
	return b.readOnly
}

func (glusterfsVolume *glusterfs) GetPath() string {
	name := glusterfsPluginName
	return glusterfsVolume.plugin.host.GetPodVolumeDir(glusterfsVolume.pod.UID, util.EscapeQualifiedNameForDisk(name), glusterfsVolume.volName)
}

type glusterfsCleaner struct {
	*glusterfs
}

var _ volume.Cleaner = &glusterfsCleaner{}

func (c *glusterfsCleaner) TearDown() error {
	return c.TearDownAt(c.GetPath())
}

func (c *glusterfsCleaner) TearDownAt(dir string) error {
	return c.cleanup(dir)
}

func (c *glusterfsCleaner) cleanup(dir string) error {
	notMnt, err := c.mounter.IsLikelyNotMountPoint(dir)
	if err != nil {
		glog.Errorf("Glusterfs: Error checking IsLikelyNotMountPoint: %v", err)
		return err
	}
	if notMnt {
		return os.RemoveAll(dir)
	}

	if err := c.mounter.Unmount(dir); err != nil {
		glog.Errorf("Glusterfs: Unmounting failed: %v", err)
		return err
	}
	notMnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir)
	if mntErr != nil {
		glog.Errorf("Glusterfs: IsLikelyNotMountPoint check failed: %v", mntErr)
		return mntErr
	}
	if notMnt {
		if err := os.RemoveAll(dir); err != nil {
			return err
		}
	}

	return nil
}

func (b *glusterfsBuilder) setUpAtInternal(dir string) error {
	var errs error

	options := []string{}
	if b.readOnly {
		options = append(options, "ro")
	}

	l := len(b.hosts.Subsets)
	// Avoid mount storm, pick a host randomly.
	start := rand.Int() % l
	// Iterate all hosts until mount succeeds.
	for i := start; i < start+l; i++ {
		hostIP := b.hosts.Subsets[i%l].Addresses[0].IP
		errs = b.mounter.Mount(hostIP+":"+b.path, dir, "glusterfs", options)
		if errs == nil {
			return nil
		}
	}
	glog.Errorf("Glusterfs: mount failed: %v", errs)
	return errs
}
<commit_msg>Adding random seed to gluster.go per #13125<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package glusterfs

import (
	"math/rand"
    "time"
	"os"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/volume"
)

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&glusterfsPlugin{nil}}
}

type glusterfsPlugin struct {
	host volume.VolumeHost
}

var _ volume.VolumePlugin = &glusterfsPlugin{}
var _ volume.PersistentVolumePlugin = &glusterfsPlugin{}

const (
	glusterfsPluginName = "kubernetes.io/glusterfs"
)

func (plugin *glusterfsPlugin) Init(host volume.VolumeHost) {
	plugin.host = host
}

func (plugin *glusterfsPlugin) Name() string {
	return glusterfsPluginName
}

func (plugin *glusterfsPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Glusterfs != nil) ||
		(spec.Volume != nil && spec.Volume.Glusterfs != nil)
}

func (plugin *glusterfsPlugin) GetAccessModes() []api.PersistentVolumeAccessMode {
	return []api.PersistentVolumeAccessMode{
		api.ReadWriteOnce,
		api.ReadOnlyMany,
		api.ReadWriteMany,
	}
}

func (plugin *glusterfsPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, _ volume.VolumeOptions, mounter mount.Interface) (volume.Builder, error) {
	source, _ := plugin.getGlusterVolumeSource(spec)
	ep_name := source.EndpointsName
	ns := pod.Namespace
	ep, err := plugin.host.GetKubeClient().Endpoints(ns).Get(ep_name)
	if err != nil {
		glog.Errorf("Glusterfs: failed to get endpoints %s[%v]", ep_name, err)
		return nil, err
	}
	glog.V(1).Infof("Glusterfs: endpoints %v", ep)
	return plugin.newBuilderInternal(spec, ep, pod, mounter, exec.New())
}

func (plugin *glusterfsPlugin) getGlusterVolumeSource(spec *volume.Spec) (*api.GlusterfsVolumeSource, bool) {
	// Glusterfs volumes used directly in a pod have a ReadOnly flag set by the pod author.
	// Glusterfs volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
	if spec.Volume != nil && spec.Volume.Glusterfs != nil {
		return spec.Volume.Glusterfs, spec.Volume.Glusterfs.ReadOnly
	} else {
		return spec.PersistentVolume.Spec.Glusterfs, spec.ReadOnly
	}
}

func (plugin *glusterfsPlugin) newBuilderInternal(spec *volume.Spec, ep *api.Endpoints, pod *api.Pod, mounter mount.Interface, exe exec.Interface) (volume.Builder, error) {
	source, readOnly := plugin.getGlusterVolumeSource(spec)
	return &glusterfsBuilder{
		glusterfs: &glusterfs{
			volName: spec.Name(),
			mounter: mounter,
			pod:     pod,
			plugin:  plugin,
		},
		hosts:    ep,
		path:     source.Path,
		readOnly: readOnly,
		exe:      exe}, nil
}

func (plugin *glusterfsPlugin) NewCleaner(volName string, podUID types.UID, mounter mount.Interface) (volume.Cleaner, error) {
	return plugin.newCleanerInternal(volName, podUID, mounter)
}

func (plugin *glusterfsPlugin) newCleanerInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Cleaner, error) {
	return &glusterfsCleaner{&glusterfs{
		volName: volName,
		mounter: mounter,
		pod:     &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
		plugin:  plugin,
	}}, nil
}

// Glusterfs volumes represent a bare host file or directory mount of an Glusterfs export.
type glusterfs struct {
	volName string
	pod     *api.Pod
	mounter mount.Interface
	plugin  *glusterfsPlugin
}

type glusterfsBuilder struct {
	*glusterfs
	hosts    *api.Endpoints
	path     string
	readOnly bool
	exe      exec.Interface
}

var _ volume.Builder = &glusterfsBuilder{}

// SetUp attaches the disk and bind mounts to the volume path.
func (b *glusterfsBuilder) SetUp() error {
	return b.SetUpAt(b.GetPath())
}

func (b *glusterfsBuilder) SetUpAt(dir string) error {
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	glog.V(4).Infof("Glusterfs: mount set up: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		return nil
	}

	os.MkdirAll(dir, 0750)
	err = b.setUpAtInternal(dir)
	if err == nil {
		return nil
	}

	// Cleanup upon failure.
	c := &glusterfsCleaner{b.glusterfs}
	c.cleanup(dir)
	return err
}

func (b *glusterfsBuilder) IsReadOnly() bool {
	return b.readOnly
}

func (glusterfsVolume *glusterfs) GetPath() string {
	name := glusterfsPluginName
	return glusterfsVolume.plugin.host.GetPodVolumeDir(glusterfsVolume.pod.UID, util.EscapeQualifiedNameForDisk(name), glusterfsVolume.volName)
}

type glusterfsCleaner struct {
	*glusterfs
}

var _ volume.Cleaner = &glusterfsCleaner{}

func (c *glusterfsCleaner) TearDown() error {
	return c.TearDownAt(c.GetPath())
}

func (c *glusterfsCleaner) TearDownAt(dir string) error {
	return c.cleanup(dir)
}

func (c *glusterfsCleaner) cleanup(dir string) error {
	notMnt, err := c.mounter.IsLikelyNotMountPoint(dir)
	if err != nil {
		glog.Errorf("Glusterfs: Error checking IsLikelyNotMountPoint: %v", err)
		return err
	}
	if notMnt {
		return os.RemoveAll(dir)
	}

	if err := c.mounter.Unmount(dir); err != nil {
		glog.Errorf("Glusterfs: Unmounting failed: %v", err)
		return err
	}
	notMnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir)
	if mntErr != nil {
		glog.Errorf("Glusterfs: IsLikelyNotMountPoint check failed: %v", mntErr)
		return mntErr
	}
	if notMnt {
		if err := os.RemoveAll(dir); err != nil {
			return err
		}
	}

	return nil
}

func (b *glusterfsBuilder) setUpAtInternal(dir string) error {
	var errs error

	options := []string{}
	if b.readOnly {
		options = append(options, "ro")
	}

	l := len(b.hosts.Subsets)
	// Avoid mount storm, pick a host randomly.
    rand.Seed(time.Now().UnixNano())
	start := rand.Int() % l
	// Iterate all hosts until mount succeeds.
	for i := start; i < start+l; i++ {
		hostIP := b.hosts.Subsets[i%l].Addresses[0].IP
		errs = b.mounter.Mount(hostIP+":"+b.path, dir, "glusterfs", options)
		if errs == nil {
			return nil
		}
	}
	glog.Errorf("Glusterfs: mount failed: %v", errs)
	return errs
}
<|endoftext|>


<commit_before>package runtime

// Version is the current version of the buffalo binary
const Version = "v0.14.11"
<commit_msg>version bump: v0.14.12<commit_after>package runtime

// Version is the current version of the buffalo binary
const Version = "v0.14.12"
<|endoftext|>


<commit_before>package models

import (
	"bytes"
	"database/sql"
	"fmt"
	"strconv"
	"strings"
)

// SearchOptions are options that can be passed to SearchSets for filtering
// sets.
type SearchOptions struct {
	// If len is 0, then it should be treated as if all statuses are good.
	Status []int
	Query  string
	// Gamemodes to which limit the results. If len is 0, it means all modes
	// are ok.
	Mode []int

	// Pagination options.
	Offset int
	Amount int
}

func (o SearchOptions) setModes() (total uint8) {
	for _, m := range o.Mode {
		if m < 0 || m >= 4 {
			continue
		}
		total |= 1 << uint8(m)
	}
	return
}

var mysqlStringReplacer = strings.NewReplacer(
	`\`, `\\`,
	`"`, `\"`,
	`'`, `\'`,
	"\x00", `\0`,
	"\n", `\n`,
	"\r", `\r`,
	"\x1a", `\Z`,
)

func sIntCommaSeparated(nums []int) string {
	b := bytes.Buffer{}
	for idx, num := range nums {
		b.WriteString(strconv.Itoa(num))
		if idx != len(nums)-1 {
			b.WriteString(", ")
		}
	}
	return b.String()
}

// SearchSets retrieves sets, filtering them using SearchOptions.
func SearchSets(db, searchDB *sql.DB, opts SearchOptions) ([]Set, error) {
	sm := strconv.Itoa(int(opts.setModes()))
	setIDsQuery := "SELECT id, set_modes & " + sm + " AS valid_set_modes FROM cg WHERE "

	// add filters to query
	// Yes. I know. Prepared statements. But Sphinx doesn't like them, so
	// bummer.
	setIDsQuery += "MATCH('" + mysqlStringReplacer.Replace(opts.Query) + "') "
	if len(opts.Status) != 0 {
		setIDsQuery += "AND ranked_status IN (" + sIntCommaSeparated(opts.Status) + ") "
	}
	if len(opts.Mode) != 0 {
		// This is a hack. Apparently, Sphinx does not support AND bitwise
		// operations in the WHERE clause, so we're placing that in the SELECT
		// clause and only making sure it's correct in this place.
		setIDsQuery += "AND valid_set_modes = " + sm + " "
	}

	// set limit
	setIDsQuery += fmt.Sprintf("ORDER BY WEIGHT() DESC, id DESC LIMIT %d, %d OPTION ranker=sph04, max_matches=10000000", opts.Offset, opts.Amount)

	// fetch rows
	rows, err := searchDB.Query(setIDsQuery)
	if err != nil {
		return nil, err
	}

	// from the rows we will retrieve the IDs of all our sets.
	// we also pre-create the slices containing the sets we will fill later on
	// when we fetch the actual data.
	setIDs := make([]int, 0, opts.Amount)
	sets := make([]Set, 0, opts.Amount)
	// setMap, having an ID, points to a position of a set contained in sets.
	setMap := make(map[int]int, opts.Amount)
	for rows.Next() {
		var id int
		err = rows.Scan(&id, new(int))
		if err != nil {
			return nil, err
		}
		setIDs = append(setIDs, id)
		sets = append(sets, Set{})
		setMap[id] = len(sets) - 1
	}

	// short circuit: there are no sets
	if len(sets) == 0 {
		return []Set{}, nil
	}

	setsQuery := "SELECT " + setFields + " FROM sets WHERE id IN (" + inClause(len(setIDs)) + ")"
	args := sIntToSInterface(setIDs)

	rows, err = db.Query(setsQuery, args...)

	if err != nil {
		return nil, err
	}

	// find all beatmaps, but leave children aside for the moment.
	for rows.Next() {
		var s Set
		err = rows.Scan(
			&s.ID, &s.RankedStatus, &s.ApprovedDate, &s.LastUpdate, &s.LastChecked,
			&s.Artist, &s.Title, &s.Creator, &s.Source, &s.Tags, &s.HasVideo, &s.Genre,
			&s.Language, &s.Favourites,
		)
		if err != nil {
			return nil, err
		}
		sets[setMap[s.ID]] = s
	}

	rows, err = db.Query(
		"SELECT "+beatmapFields+" FROM beatmaps WHERE parent_set_id IN ("+
			inClause(len(setIDs))+")",
		sIntToSInterface(setIDs)...,
	)
	if err != nil {
		return nil, err
	}

	for rows.Next() {
		var b Beatmap
		err = rows.Scan(
			&b.ID, &b.ParentSetID, &b.DiffName, &b.FileMD5, &b.Mode, &b.BPM,
			&b.AR, &b.OD, &b.CS, &b.HP, &b.TotalLength, &b.HitLength,
			&b.Playcount, &b.Passcount, &b.MaxCombo, &b.DifficultyRating,
		)
		if err != nil {
			return nil, err
		}
		parentSet, ok := setMap[b.ParentSetID]
		if !ok {
			continue
		}
		sets[parentSet].ChildrenBeatmaps = append(sets[parentSet].ChildrenBeatmaps, b)
	}

	return sets, nil
}
<commit_msg>Query mysql straight when no text query is given<commit_after>package models

import (
	"bytes"
	"database/sql"
	"fmt"
	"strconv"
	"strings"
)

// SearchOptions are options that can be passed to SearchSets for filtering
// sets.
type SearchOptions struct {
	// If len is 0, then it should be treated as if all statuses are good.
	Status []int
	Query  string
	// Gamemodes to which limit the results. If len is 0, it means all modes
	// are ok.
	Mode []int

	// Pagination options.
	Offset int
	Amount int
}

func (o SearchOptions) setModes() (total uint8) {
	for _, m := range o.Mode {
		if m < 0 || m >= 4 {
			continue
		}
		total |= 1 << uint8(m)
	}
	return
}

var mysqlStringReplacer = strings.NewReplacer(
	`\`, `\\`,
	`"`, `\"`,
	`'`, `\'`,
	"\x00", `\0`,
	"\n", `\n`,
	"\r", `\r`,
	"\x1a", `\Z`,
)

func sIntCommaSeparated(nums []int) string {
	b := bytes.Buffer{}
	for idx, num := range nums {
		b.WriteString(strconv.Itoa(num))
		if idx != len(nums)-1 {
			b.WriteString(", ")
		}
	}
	return b.String()
}

// SearchSets retrieves sets, filtering them using SearchOptions.
func SearchSets(db, searchDB *sql.DB, opts SearchOptions) ([]Set, error) {
	sm := strconv.Itoa(int(opts.setModes()))

	// first, we create the where conditions that are valid for both querying mysql
	// straight or querying sphinx first.
	var whereConds string
	var havingConds string
	if len(opts.Status) != 0 {
		whereConds = "ranked_status IN (" + sIntCommaSeparated(opts.Status) + ") "
	}
	if len(opts.Mode) != 0 {
		// This is a hack. Apparently, Sphinx does not support AND bitwise
		// operations in the WHERE clause, so we're placing that in the SELECT
		// clause and only making sure it's correct in this place.
		havingConds = " valid_set_modes = " + sm + " "
	}

	sets := make([]Set, 0, opts.Amount)
	setIDs := make([]int, 0, opts.Amount)
	// setMap is used when a query is given to make sure the results are kept in the correct
	// order given by sphinx.
	setMap := make(map[int]int, opts.Amount)
	// if Sphinx is used, limit will be cleared so that it's not used for the mysql query
	limit := fmt.Sprintf(" LIMIT %d, %d ", opts.Offset, opts.Amount)

	if opts.Query != "" {
		setIDsQuery := "SELECT id, set_modes & " + sm + " AS valid_set_modes FROM cg WHERE "

		// add filters to query
		// Yes. I know. Prepared statements. But Sphinx doesn't like them, so
		// bummer.
		setIDsQuery += "MATCH('" + mysqlStringReplacer.Replace(opts.Query) + "') "
		if whereConds != "" {
			setIDsQuery += "AND " + whereConds
		}
		if havingConds != "" {
			setIDsQuery += " AND " + havingConds
		}

		// set limit
		setIDsQuery += " ORDER BY WEIGHT() DESC, id DESC " + limit + " OPTION ranker=sph04, max_matches=20000 "
		limit = ""

		// fetch rows
		rows, err := searchDB.Query(setIDsQuery)
		if err != nil {
			return nil, err
		}

		// contains IDs of the sets we will retrieve
		for rows.Next() {
			var id int
			err = rows.Scan(&id, new(int))
			if err != nil {
				return nil, err
			}
			setIDs = append(setIDs, id)
			sets = sets[:len(sets)+1]
			setMap[id] = len(sets) - 1
		}

		// short path: there are no sets
		if len(sets) == 0 {
			return []Set{}, nil
		}

		whereConds = "id IN (" + sIntCommaSeparated(setIDs) + ")"
		havingConds = ""
	}

	if whereConds != "" {
		whereConds = "WHERE " + whereConds
	}
	if havingConds != "" {
		havingConds = " HAVING " + havingConds
	}
	setsQuery := "SELECT " + setFields + ", set_modes & " + sm + " AS valid_set_modes FROM sets " +
		whereConds + havingConds + " ORDER BY id DESC " + limit
	rows, err := db.Query(setsQuery)

	if err != nil {
		return nil, err
	}

	// find all beatmaps, but leave children aside for the moment.
	for rows.Next() {
		var s Set
		err = rows.Scan(
			&s.ID, &s.RankedStatus, &s.ApprovedDate, &s.LastUpdate, &s.LastChecked,
			&s.Artist, &s.Title, &s.Creator, &s.Source, &s.Tags, &s.HasVideo, &s.Genre,
			&s.Language, &s.Favourites, new(int),
		)
		if err != nil {
			return nil, err
		}
		// we get the position we should place s in from the setMap, this way we
		// keep the order of results as sphinx prefers.
		pos, ok := setMap[s.ID]
		if ok {
			sets[pos] = s
		} else {
			sets = append(sets, s)
			setIDs = append(setIDs, s.ID)
			setMap[s.ID] = len(sets) - 1
		}
	}

	if len(sets) == 0 {
		return sets, nil
	}

	rows, err = db.Query(
		"SELECT "+beatmapFields+" FROM beatmaps WHERE parent_set_id IN ("+
			inClause(len(setIDs))+")",
		sIntToSInterface(setIDs)...,
	)
	if err != nil {
		return nil, err
	}

	for rows.Next() {
		var b Beatmap
		err = rows.Scan(
			&b.ID, &b.ParentSetID, &b.DiffName, &b.FileMD5, &b.Mode, &b.BPM,
			&b.AR, &b.OD, &b.CS, &b.HP, &b.TotalLength, &b.HitLength,
			&b.Playcount, &b.Passcount, &b.MaxCombo, &b.DifficultyRating,
		)
		if err != nil {
			return nil, err
		}
		parentSet, ok := setMap[b.ParentSetID]
		if !ok {
			continue
		}
		sets[parentSet].ChildrenBeatmaps = append(sets[parentSet].ChildrenBeatmaps, b)
	}

	return sets, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth

import (
	"time"

	"github.com/gravitational/teleport/lib/events"
	"github.com/gravitational/teleport/lib/services"

	"github.com/codahale/lunk"
)

// AccessPoint is a interface needed by nodes to control the access
// to the node, and provide heartbeats
type AccessPoint interface {

	// GetLocalDomain returns domain name of the local authority server
	GetLocalDomain() (string, error)

	// GetServers returns a list of registered servers
	GetNodes() ([]services.Server, error)

	// UpsertServer registers server presence, permanently if ttl is 0 or
	// for the specified duration with second resolution if it's >= 1 second
	UpsertNode(s services.Server, ttl time.Duration) error

	// UpsertProxy registers server presence, permanently if ttl is 0 or
	// for the specified duration with second resolution if it's >= 1 second
	UpsertProxy(s services.Server, ttl time.Duration) error

	// GetProxies returns a list of proxy servers registered in the cluster
	GetProxies() ([]services.Server, error)

	// GetCertAuthorities returns a list of cert authorities
	GetCertAuthorities(caType services.CertAuthType, loadKeys bool) ([]*services.CertAuthority, error)

	// GetUsers returns a list of local users registered with this domain
	GetUsers() ([]services.User, error)

	// GetEvents returns a list of events that
	GetEvents(filter events.Filter) ([]lunk.Entry, error)
}
<commit_msg>comments<commit_after>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package auth

import (
	"time"

	"github.com/gravitational/teleport/lib/events"
	"github.com/gravitational/teleport/lib/services"

	"github.com/codahale/lunk"
)

// AccessPoint is a interface needed by nodes to control the access
// to the node, and provide heartbeats
type AccessPoint interface {

	// GetLocalDomain returns domain name of the local authority server
	GetLocalDomain() (string, error)

	// GetServers returns a list of registered servers
	GetNodes() ([]services.Server, error)

	// UpsertServer registers server presence, permanently if ttl is 0 or
	// for the specified duration with second resolution if it's >= 1 second
	UpsertNode(s services.Server, ttl time.Duration) error

	// UpsertProxy registers server presence, permanently if ttl is 0 or
	// for the specified duration with second resolution if it's >= 1 second
	UpsertProxy(s services.Server, ttl time.Duration) error

	// GetProxies returns a list of proxy servers registered in the cluster
	GetProxies() ([]services.Server, error)

	// GetCertAuthorities returns a list of cert authorities
	GetCertAuthorities(caType services.CertAuthType, loadKeys bool) ([]*services.CertAuthority, error)

	// GetUsers returns a list of local users registered with this domain
	GetUsers() ([]services.User, error)

	// GetEvents returns a list of events that
	GetEvents(filter events.Filter) ([]lunk.Entry, error)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build ignore

package main

// print-mpb-powers-of-10.go prints the medium-precision (128 bit mantissa)
// binary (base 2) wuffs_base__private_implementation__powers_of_10 tables.
//
// Usage: go run print-mpb-powers-of-10.go -detail

import (
	"flag"
	"fmt"
	"math/big"
	"os"
)

var (
	detail = flag.Bool("detail", false, "whether to print detailed comments")
)

func main() {
	if err := main1(); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
}

func main1() error {
	flag.Parse()

	const count = 1 + (+310 - -326)
	fmt.Printf("static const uint32_t "+
		"wuffs_base__private_implementation__powers_of_10[%d] = {\n", 5*count)
	for e := -326; e <= +310; e++ {
		if err := do(e); err != nil {
			return err
		}
	}
	fmt.Printf("};\n\n")

	return nil
}

var (
	one    = big.NewInt(1)
	ten    = big.NewInt(10)
	two128 = big.NewInt(0).Lsh(one, 128)
)

// N is large enough so that (1<<N) is easily bigger than 1e310.
const N = 2048

// 1214 is 1023 + 191. 1023 is the bias for IEEE 754 double-precision floating
// point. 191 is ((3 * 64) - 1) and we work with multiples-of-64-bit mantissas.
const bias = 1214

func do(e int) error {
	z := big.NewInt(0).Lsh(one, N)
	if e >= 0 {
		exp := big.NewInt(0).Exp(ten, big.NewInt(int64(+e)), nil)
		z.Mul(z, exp)
	} else {
		exp := big.NewInt(0).Exp(ten, big.NewInt(int64(-e)), nil)
		z.Div(z, exp)
	}

	n := int32(-N)
	for z.Cmp(two128) >= 0 {
		z.Rsh(z, 1)
		n++
	}
	hex := fmt.Sprintf("%X", z)
	if len(hex) != 32 {
		return fmt.Errorf("invalid hexadecimal representation %q", hex)
	}

	fmt.Printf("    0x%s, 0x%s, 0x%s, 0x%s, 0x%04X,  // 1e%-04d",
		hex[24:], hex[16:24], hex[8:16], hex[:8], uint32(n)+bias, e)
	if *detail {
		fmt.Printf("≈ (0x%s ", e, hex)
		if n >= 0 {
			fmt.Printf("<< %4d)", +n)
		} else {
			fmt.Printf(">> %4d)", -n)
		}
	}

	fmt.Println()
	return nil
}
<commit_msg>Fix script/print-mpb-powers-of-10.go with -detail<commit_after>// Copyright 2020 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build ignore

package main

// print-mpb-powers-of-10.go prints the medium-precision (128-bit mantissa)
// binary (base-2) wuffs_base__private_implementation__powers_of_10 tables.
//
// When the approximation to (10 ** N) is not exact, the mantissa is truncated,
// not rounded to nearest. The base-2 exponent is chosen so that the mantissa's
// most signficant bit (bit 127) is set.
//
// The final uint32_t entry in each row-of-5 is biased by 1214, also known as
// 0x04BE, which simplifies its use in f64conv-submodule.c.
//
// Usage: go run print-mpb-powers-of-10.go -detail
//
// With -detail set, its output should include:
//
// 0xF7604B57, 0x014BB630, 0xFE98746D, 0x84A57695, 0x0004,
//    // 1e-326 ≈ (0x84A57695FE98746D014BB630F7604B57 >> 1210)
//
// 0x35385E2D, 0x419EA3BD, 0x7E3E9188, 0xA5CED43B, 0x0007,
//    // 1e-325 ≈ (0xA5CED43B7E3E9188419EA3BD35385E2D >> 1207)
//
// ...
//
// 0x0A3D70A3, 0x3D70A3D7, 0x70A3D70A, 0xA3D70A3D, 0x0438,
//    // 1e-2   ≈ (0xA3D70A3D70A3D70A3D70A3D70A3D70A3 >>  134)
//
// 0xCCCCCCCC, 0xCCCCCCCC, 0xCCCCCCCC, 0xCCCCCCCC, 0x043B,
//    // 1e-1   ≈ (0xCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC >>  131)
//
// 0x00000000, 0x00000000, 0x00000000, 0x80000000, 0x043F,
//    // 1e0    ≈ (0x80000000000000000000000000000000 >>  127)
//
// 0x00000000, 0x00000000, 0x00000000, 0xA0000000, 0x0442,
//    // 1e1    ≈ (0xA0000000000000000000000000000000 >>  124)
//
// 0x00000000, 0x00000000, 0x00000000, 0xC8000000, 0x0445,
//    // 1e2    ≈ (0xC8000000000000000000000000000000 >>  121)
//
// ...
//
// 0x51E513DA, 0x2CD2CC65, 0x35D63F73, 0xB201833B, 0x0841,
//    // 1e309  ≈ (0xB201833B35D63F732CD2CC6551E513DA <<  899)
//
// 0xA65E58D1, 0xF8077F7E, 0x034BCF4F, 0xDE81E40A, 0x0844,
//    // 1e310  ≈ (0xDE81E40A034BCF4FF8077F7EA65E58D1 <<  902)

import (
	"flag"
	"fmt"
	"math/big"
	"os"
)

var (
	detail = flag.Bool("detail", false, "whether to print detailed comments")
)

func main() {
	if err := main1(); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
}

func main1() error {
	flag.Parse()

	const count = 1 + (+310 - -326)
	fmt.Printf("static const uint32_t "+
		"wuffs_base__private_implementation__powers_of_10[%d] = {\n", 5*count)
	for e := -326; e <= +310; e++ {
		if err := do(e); err != nil {
			return err
		}
	}
	fmt.Printf("};\n\n")

	return nil
}

var (
	one    = big.NewInt(1)
	ten    = big.NewInt(10)
	two128 = big.NewInt(0).Lsh(one, 128)
)

// N is large enough so that (1<<N) is easily bigger than 1e310.
const N = 2048

// 1214 is 1023 + 191. 1023 is the bias for IEEE 754 double-precision floating
// point. 191 is ((3 * 64) - 1) and we work with multiples-of-64-bit mantissas.
const bias = 1214

func do(e int) error {
	z := big.NewInt(0).Lsh(one, N)
	if e >= 0 {
		exp := big.NewInt(0).Exp(ten, big.NewInt(int64(+e)), nil)
		z.Mul(z, exp)
	} else {
		exp := big.NewInt(0).Exp(ten, big.NewInt(int64(-e)), nil)
		z.Div(z, exp)
	}

	n := int32(-N)
	for z.Cmp(two128) >= 0 {
		z.Rsh(z, 1)
		n++
	}
	hex := fmt.Sprintf("%X", z)
	if len(hex) != 32 {
		return fmt.Errorf("invalid hexadecimal representation %q", hex)
	}

	fmt.Printf("    0x%s, 0x%s, 0x%s, 0x%s, 0x%04X,  // 1e%-04d",
		hex[24:], hex[16:24], hex[8:16], hex[:8], uint32(n)+bias, e)
	if *detail {
		fmt.Printf(" ≈ (0x%s ", hex)
		if n >= 0 {
			fmt.Printf("<< %4d)", +n)
		} else {
			fmt.Printf(">> %4d)", -n)
		}
	}

	fmt.Println()
	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package fs

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/syncthing/syncthing/lib/build"
)

const pathSeparatorString = string(PathSeparator)

func ExpandTilde(path string) (string, error) {
	if path == "~" {
		return getHomeDir()
	}

	path = filepath.FromSlash(path)
	if !strings.HasPrefix(path, fmt.Sprintf("~%c", PathSeparator)) {
		return path, nil
	}

	home, err := getHomeDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(home, path[2:]), nil
}

func getHomeDir() (string, error) {
	if build.IsWindows {
		// Legacy -- we prioritize this for historical reasons, whereas
		// os.UserHomeDir uses %USERPROFILE% always.
		home := filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home != "" {
			return home, nil
		}
	}

	return os.UserHomeDir()
}

const windowsDisallowedCharacters = (`<>:"|?*` +
	"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" +
	"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f")

func WindowsInvalidFilename(name string) error {
	// None of the path components should end in space or period, or be a
	// reserved name.
	for _, part := range strings.Split(name, `\`) {
		if part == "" {
			continue
		}
		switch part[len(part)-1] {
		case ' ', '.':
			// Names ending in space or period are not valid.
			return errInvalidFilenameWindowsSpacePeriod
		}
		if windowsIsReserved(part) {
			return errInvalidFilenameWindowsReservedName
		}
	}

	// The path must not contain any disallowed characters
	if strings.ContainsAny(name, windowsDisallowedCharacters) {
		return errInvalidFilenameWindowsReservedChar
	}

	return nil
}

// SanitizePath takes a string that might contain all kinds of special
// characters and makes a valid, similar, path name out of it.
//
// Spans of invalid characters, whitespace and/or non-UTF-8 sequences are
// replaced by a single space. The result is always UTF-8 and contains only
// printable characters, as determined by unicode.IsPrint.
//
// Invalid characters are non-printing runes, things not allowed in file names
// in Windows, and common shell metacharacters. Even if asterisks and pipes
// and stuff are allowed on Unixes in general they might not be allowed by
// the filesystem and may surprise the user and cause shell oddness. This
// function is intended for file names we generate on behalf of the user,
// and surprising them with odd shell characters in file names is unkind.
//
// We include whitespace in the invalid characters so that multiple
// whitespace is collapsed to a single space. Additionally, whitespace at
// either end is removed.
//
// If the result is a name disallowed on windows, a hyphen is prepended.
func SanitizePath(path string) string {
	var b strings.Builder

	const disallowed = `'/\[]{};:!@$%&^#` + windowsDisallowedCharacters
	prev := ' '
	for _, c := range path {
		if !unicode.IsPrint(c) || c == unicode.ReplacementChar ||
			strings.ContainsRune(disallowed, c) {
			c = ' '
		}

		if !(c == ' ' && prev == ' ') {
			b.WriteRune(c)
		}
		prev = c
	}

	path = strings.TrimSpace(b.String())
	if windowsIsReserved(path) {
		path = "-" + path
	}
	return path
}

func windowsIsReserved(part string) bool {
	// nul.txt.jpg is also disallowed.
	dot := strings.IndexByte(part, '.')
	if dot != -1 {
		part = part[:dot]
	}

	// Check length to skip allocating ToUpper.
	if len(part) != 3 && len(part) != 4 {
		return false
	}

	// COM0 and LPT0 are missing from the Microsoft docs,
	// but Windows Explorer treats them as invalid too.
	// (https://docs.microsoft.com/windows/win32/fileio/naming-a-file)
	switch strings.ToUpper(part) {
	case "CON", "PRN", "AUX", "NUL",
		"COM0", "COM1", "COM2", "COM3", "COM4",
		"COM5", "COM6", "COM7", "COM8", "COM9",
		"LPT0", "LPT1", "LPT2", "LPT3", "LPT4",
		"LPT5", "LPT6", "LPT7", "LPT8", "LPT9":
		return true
	}
	return false
}

// IsParent compares paths purely lexicographically, meaning it returns false
// if path and parent aren't both absolute or relative.
func IsParent(path, parent string) bool {
	if parent == path {
		// Twice the same root on windows would not be caught at the end.
		return false
	}
	if filepath.IsAbs(path) != filepath.IsAbs(parent) {
		return false
	}
	if parent == "" || parent == "." {
		// The empty string is the parent of everything except the empty
		// string and ".". (Avoids panic in the last step.)
		return path != "" && path != "."
	}
	if parent == "/" {
		// The root is the parent of everything except itself, which would
		// not be caught below.
		return path != "/"
	}
	if parent[len(parent)-1] != PathSeparator {
		parent += pathSeparatorString
	}
	return strings.HasPrefix(path, parent)
}

func CommonPrefix(first, second string) string {
	if filepath.IsAbs(first) != filepath.IsAbs(second) {
		// Whatever
		return ""
	}

	firstParts := PathComponents(filepath.Clean(first))
	secondParts := PathComponents(filepath.Clean(second))

	isAbs := filepath.IsAbs(first) && filepath.IsAbs(second)

	count := len(firstParts)
	if len(secondParts) < len(firstParts) {
		count = len(secondParts)
	}

	common := make([]string, 0, count)
	for i := 0; i < count; i++ {
		if firstParts[i] != secondParts[i] {
			break
		}
		common = append(common, firstParts[i])
	}

	if isAbs {
		if build.IsWindows && isVolumeNameOnly(common) {
			// Because strings.Split strips out path separators, if we're at the volume name, we end up without a separator
			// Wedge an empty element to be joined with.
			common = append(common, "")
		} else if len(common) == 1 {
			// If isAbs on non Windows, first element in both first and second is "", hence joining that returns nothing.
			return pathSeparatorString
		}
	}

	// This should only be true on Windows when drive letters are different or when paths are relative.
	// In case of UNC paths we should end up with more than a single element hence joining is fine
	if len(common) == 0 {
		return ""
	}

	// This has to be strings.Join, because filepath.Join([]string{"", "", "?", "C:", "Audrius"}...) returns garbage
	result := strings.Join(common, pathSeparatorString)
	return filepath.Clean(result)
}

// PathComponents returns a list of names of parent directories and the leaf
// item for the given native (fs.PathSeparator delimited) and clean path.
func PathComponents(path string) []string {
	return strings.Split(path, pathSeparatorString)
}

func isVolumeNameOnly(parts []string) bool {
	isNormalVolumeName := len(parts) == 1 && strings.HasSuffix(parts[0], ":")
	isUNCVolumeName := len(parts) == 4 && strings.HasSuffix(parts[3], ":")
	return isNormalVolumeName || isUNCVolumeName
}
<commit_msg>lib/fs: Optimize WindowsInvalidFilename (#8687)<commit_after>// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package fs

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/syncthing/syncthing/lib/build"
)

const pathSeparatorString = string(PathSeparator)

func ExpandTilde(path string) (string, error) {
	if path == "~" {
		return getHomeDir()
	}

	path = filepath.FromSlash(path)
	if !strings.HasPrefix(path, fmt.Sprintf("~%c", PathSeparator)) {
		return path, nil
	}

	home, err := getHomeDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(home, path[2:]), nil
}

func getHomeDir() (string, error) {
	if build.IsWindows {
		// Legacy -- we prioritize this for historical reasons, whereas
		// os.UserHomeDir uses %USERPROFILE% always.
		home := filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home != "" {
			return home, nil
		}
	}

	return os.UserHomeDir()
}

const windowsDisallowedCharacters = (`<>:"|?*` +
	"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" +
	"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f")

func WindowsInvalidFilename(name string) error {
	// The path must not contain any disallowed characters.
	if strings.ContainsAny(name, windowsDisallowedCharacters) {
		return errInvalidFilenameWindowsReservedChar
	}

	// None of the path components should end in space or period, or be a
	// reserved name.
	for len(name) > 0 {
		part, rest, _ := strings.Cut(name, `\`)
		name = rest

		if part == "" {
			continue
		}
		switch part[len(part)-1] {
		case ' ', '.':
			// Names ending in space or period are not valid.
			return errInvalidFilenameWindowsSpacePeriod
		}
		if windowsIsReserved(part) {
			return errInvalidFilenameWindowsReservedName
		}
	}

	return nil
}

// SanitizePath takes a string that might contain all kinds of special
// characters and makes a valid, similar, path name out of it.
//
// Spans of invalid characters, whitespace and/or non-UTF-8 sequences are
// replaced by a single space. The result is always UTF-8 and contains only
// printable characters, as determined by unicode.IsPrint.
//
// Invalid characters are non-printing runes, things not allowed in file names
// in Windows, and common shell metacharacters. Even if asterisks and pipes
// and stuff are allowed on Unixes in general they might not be allowed by
// the filesystem and may surprise the user and cause shell oddness. This
// function is intended for file names we generate on behalf of the user,
// and surprising them with odd shell characters in file names is unkind.
//
// We include whitespace in the invalid characters so that multiple
// whitespace is collapsed to a single space. Additionally, whitespace at
// either end is removed.
//
// If the result is a name disallowed on windows, a hyphen is prepended.
func SanitizePath(path string) string {
	var b strings.Builder

	const disallowed = `'/\[]{};:!@$%&^#` + windowsDisallowedCharacters
	prev := ' '
	for _, c := range path {
		if !unicode.IsPrint(c) || c == unicode.ReplacementChar ||
			strings.ContainsRune(disallowed, c) {
			c = ' '
		}

		if !(c == ' ' && prev == ' ') {
			b.WriteRune(c)
		}
		prev = c
	}

	path = strings.TrimSpace(b.String())
	if windowsIsReserved(path) {
		path = "-" + path
	}
	return path
}

func windowsIsReserved(part string) bool {
	// nul.txt.jpg is also disallowed.
	dot := strings.IndexByte(part, '.')
	if dot != -1 {
		part = part[:dot]
	}

	// Check length to skip allocating ToUpper.
	if len(part) != 3 && len(part) != 4 {
		return false
	}

	// COM0 and LPT0 are missing from the Microsoft docs,
	// but Windows Explorer treats them as invalid too.
	// (https://docs.microsoft.com/windows/win32/fileio/naming-a-file)
	switch strings.ToUpper(part) {
	case "CON", "PRN", "AUX", "NUL",
		"COM0", "COM1", "COM2", "COM3", "COM4",
		"COM5", "COM6", "COM7", "COM8", "COM9",
		"LPT0", "LPT1", "LPT2", "LPT3", "LPT4",
		"LPT5", "LPT6", "LPT7", "LPT8", "LPT9":
		return true
	}
	return false
}

// IsParent compares paths purely lexicographically, meaning it returns false
// if path and parent aren't both absolute or relative.
func IsParent(path, parent string) bool {
	if parent == path {
		// Twice the same root on windows would not be caught at the end.
		return false
	}
	if filepath.IsAbs(path) != filepath.IsAbs(parent) {
		return false
	}
	if parent == "" || parent == "." {
		// The empty string is the parent of everything except the empty
		// string and ".". (Avoids panic in the last step.)
		return path != "" && path != "."
	}
	if parent == "/" {
		// The root is the parent of everything except itself, which would
		// not be caught below.
		return path != "/"
	}
	if parent[len(parent)-1] != PathSeparator {
		parent += pathSeparatorString
	}
	return strings.HasPrefix(path, parent)
}

func CommonPrefix(first, second string) string {
	if filepath.IsAbs(first) != filepath.IsAbs(second) {
		// Whatever
		return ""
	}

	firstParts := PathComponents(filepath.Clean(first))
	secondParts := PathComponents(filepath.Clean(second))

	isAbs := filepath.IsAbs(first) && filepath.IsAbs(second)

	count := len(firstParts)
	if len(secondParts) < len(firstParts) {
		count = len(secondParts)
	}

	common := make([]string, 0, count)
	for i := 0; i < count; i++ {
		if firstParts[i] != secondParts[i] {
			break
		}
		common = append(common, firstParts[i])
	}

	if isAbs {
		if build.IsWindows && isVolumeNameOnly(common) {
			// Because strings.Split strips out path separators, if we're at the volume name, we end up without a separator
			// Wedge an empty element to be joined with.
			common = append(common, "")
		} else if len(common) == 1 {
			// If isAbs on non Windows, first element in both first and second is "", hence joining that returns nothing.
			return pathSeparatorString
		}
	}

	// This should only be true on Windows when drive letters are different or when paths are relative.
	// In case of UNC paths we should end up with more than a single element hence joining is fine
	if len(common) == 0 {
		return ""
	}

	// This has to be strings.Join, because filepath.Join([]string{"", "", "?", "C:", "Audrius"}...) returns garbage
	result := strings.Join(common, pathSeparatorString)
	return filepath.Clean(result)
}

// PathComponents returns a list of names of parent directories and the leaf
// item for the given native (fs.PathSeparator delimited) and clean path.
func PathComponents(path string) []string {
	return strings.Split(path, pathSeparatorString)
}

func isVolumeNameOnly(parts []string) bool {
	isNormalVolumeName := len(parts) == 1 && strings.HasSuffix(parts[0], ":")
	isUNCVolumeName := len(parts) == 4 && strings.HasSuffix(parts[3], ":")
	return isNormalVolumeName || isUNCVolumeName
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/xyproto/recwatch"
)

// TODO: A better 404 page not found page for users visiting "/"

// TODO: Consider using channels in a more clever way, to avoid sleeping.
//       Possibly by sending channels over channels.

// TODO: Consider only listening for changes after a file has been visited, the
//       stop watching it after a while.

type (
	// TimeEventMap stores filesystem events
	TimeEventMap map[time.Time]recwatch.Event

	// For being able to sort slices of time
	timeKeys []time.Time
)

func (t timeKeys) Len() int {
	return len(t)
}

func (t timeKeys) Less(i, j int) bool {
	return t[i].Before(t[j])
}

func (t timeKeys) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}

// Event can write SSE events to the given ResponseWriter
// id can be nil.
func Event(w http.ResponseWriter, id *uint64, message string, flush bool) {
	var buf bytes.Buffer
	if id != nil {
		buf.WriteString(fmt.Sprintf("id: %v\n", *id))
	}
	for _, msg := range strings.Split(message, "\n") {
		buf.WriteString(fmt.Sprintf("data: %s\n", msg))
	}
	buf.WriteString("\n")
	io.Copy(w, &buf)
	if flush {
		Flush(w)
	}
}

// Flush can flush the given ResponseWriter.
// Returns false if it wasn't an http.Flusher.
func Flush(w http.ResponseWriter) bool {
	flusher, ok := w.(http.Flusher)
	if ok {
		flusher.Flush()
	}
	return ok
}

// Remove old events
// Must be run within a mutex
func removeOldEvents(events *TimeEventMap, maxAge time.Duration) {
	now := time.Now()
	// Cutoff time
	longTimeAgo := now.Add(-maxAge)
	// Loop through the events and delete the old ones
	for t := range *events {
		if t.Before(longTimeAgo) {
			//log.Warn("DELETING " + (*events)[t].String())
			delete(*events, t)
		}
	}
}

// Gather filesystem events in a way that web handle functions can use
func collectFileChangeEvents(watcher *recwatch.RecursiveWatcher, mut *sync.Mutex, events TimeEventMap, maxAge time.Duration) {
	go func() {
		for {
			select {
			case ev := <-watcher.Events:
				mut.Lock()
				// Remove old events
				removeOldEvents(&events, maxAge)
				// Save the event with the current time
				events[time.Now()] = recwatch.Event(ev)
				mut.Unlock()
				//log.Info(ev)
			case err := <-watcher.Errors:
				log.Error(err)
			}
		}
	}()
}

// Create events whenever a file in the server directory changes
func genFileChangeEvents(events TimeEventMap, mut *sync.Mutex, maxAge time.Duration, allowed string) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "text/event-stream;charset=utf-8")
		w.Header().Add("Cache-Control", "no-cache")
		w.Header().Add("Connection", "keep-alive")
		w.Header().Add("Access-Control-Allow-Origin", allowed)

		// TODO: Check for CloseNotify, for more graceful timeouts

		var id uint64

		for {
			func() { // Use an anonymous function, just for using "defer"
				mut.Lock()
				defer mut.Unlock()
				if len(events) > 0 {
					// Remove old keys
					removeOldEvents(&events, maxAge)
					// Sort the events by the registered time
					var keys timeKeys
					for k := range events {
						keys = append(keys, k)
					}
					sort.Sort(keys)
					prevname := ""
					for _, k := range keys {
						ev := events[k]
						if verboseMode {
							log.Info("EVENT " + ev.String())
						}
						// Avoid sending several events for the same filename
						if ev.Name != prevname {
							// Send an event to the client
							Event(w, &id, ev.Name, true)
							id++
							prevname = ev.Name
						}
					}
				}
			}()
			// Wait for old events to be gone, and new to appear
			time.Sleep(maxAge)
		}
	}
}

// EventServer serves events on a dedicated port.
// addr is the host address ([host][:port])
// urlPath is the path to handle (ie /fs)
// refresh is how often the event buffer should be checked and cleared.
// The filesystem events are gathered independently of that.
// Allowed can be "*" or a hostname and sets a header in the SSE stream.
func EventServer(addr, urlPath, path string, refresh time.Duration, allowed string) {
	// Create a new filesystem watcher
	rw, err := recwatch.NewRecursiveWatcher(path)
	if err != nil {
		fatalExit(err)
	}

	var mut sync.Mutex
	events := make(TimeEventMap)

	// Collect the events for the last n seconds, repeatedly
	// Runs in the background
	collectFileChangeEvents(rw, &mut, events, refresh)

	if strings.Contains(addr, ":") {
		fields := strings.Split(addr, ":")
		log.Info("Serving filesystem events on port " + fields[1])
	}

	// Serve events
	go func() {
		eventMux := http.NewServeMux()
		// Fire off events whenever a file in the server directory changes
		eventMux.HandleFunc(urlPath, genFileChangeEvents(events, &mut, refresh, allowed))
		eventServer := &http.Server{
			Addr:    addr,
			Handler: eventMux,
			// ReadTimeout:  3600 * time.Second,
			// WriteTimeout: 3600 *	 time.Second,
			// MaxHeaderBytes: 1 << 20,
		}
		if err := eventServer.ListenAndServe(); err != nil {
			// If we can't serve HTTP on this port, give up
			fatalExit(err)
		}
	}()
}

// Insert JavaScript that refreshes the page when the source files changes.
// The JavaScript depends on the event server being available.
// If javascript can not be inserted, return the original data.
func insertAutoRefresh(htmldata []byte) []byte {
	fullHost := eventAddr
	// If the host+port starts with ":", assume it's only the port number
	if strings.HasPrefix(fullHost, ":") {
		// Add the hostname in front
		if serverHost != "" {
			fullHost = serverHost + eventAddr
		} else {
			fullHost = "localhost" + eventAddr
		}
	}
	js := `
    `

	// TODO: Use a regexp or a JavaScript minification package instead of replacing strings

	// Reduce the size slightly
	js = strings.TrimSpace(strings.Replace(js, "\n", "", everyInstance))
	// Remove all whitespace that is more than one space
	for strings.Contains(js, "  ") {
		js = strings.Replace(js, "  ", " ", everyInstance)
	}
	// Place the script at the end of the body, if there is a body
	if bytes.Contains(htmldata, []byte("")) {
		return bytes.Replace(htmldata, []byte(""), []byte(js+""), 1)
	} else if bytes.Contains(htmldata, []byte("")) {
		// If not, place the script in the , if there is a head
		return bytes.Replace(htmldata, []byte(""), []byte(""+js), 1)
	} else if bytes.Contains(htmldata, []byte("")) {
		// If not, place the script in the  as a new
		return bytes.Replace(htmldata, []byte(""), []byte(""+js+""), 1)
	}
	// In the unlikely event that no place to insert the JavaScript was found
	return htmldata
}
<commit_msg>Smoother page reloads when auto-refresh is enabled<commit_after>package main

import (
	"bytes"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/xyproto/recwatch"
)

type (
	// TimeEventMap stores filesystem events
	TimeEventMap map[time.Time]recwatch.Event

	// For being able to sort slices of time
	timeKeys []time.Time
)

func (t timeKeys) Len() int {
	return len(t)
}

func (t timeKeys) Less(i, j int) bool {
	return t[i].Before(t[j])
}

func (t timeKeys) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}

// Event can write SSE events to the given ResponseWriter
// id can be nil.
func Event(w http.ResponseWriter, id *uint64, message string, flush bool) {
	var buf bytes.Buffer
	if id != nil {
		buf.WriteString(fmt.Sprintf("id: %v\n", *id))
	}
	for _, msg := range strings.Split(message, "\n") {
		buf.WriteString(fmt.Sprintf("data: %s\n", msg))
	}
	buf.WriteString("\n")
	io.Copy(w, &buf)
	if flush {
		Flush(w)
	}
}

// Flush can flush the given ResponseWriter.
// Returns false if it wasn't an http.Flusher.
func Flush(w http.ResponseWriter) bool {
	flusher, ok := w.(http.Flusher)
	if ok {
		flusher.Flush()
	}
	return ok
}

// Remove old events
// Must be run within a mutex
func removeOldEvents(events *TimeEventMap, maxAge time.Duration) {
	now := time.Now()
	// Cutoff time
	longTimeAgo := now.Add(-maxAge)
	// Loop through the events and delete the old ones
	for t := range *events {
		if t.Before(longTimeAgo) {
			//log.Warn("DELETING " + (*events)[t].String())
			delete(*events, t)
		}
	}
}

// Gather filesystem events in a way that web handle functions can use
func collectFileChangeEvents(watcher *recwatch.RecursiveWatcher, mut *sync.Mutex, events TimeEventMap, maxAge time.Duration) {
	go func() {
		for {
			select {
			case ev := <-watcher.Events:
				mut.Lock()
				// Remove old events
				removeOldEvents(&events, maxAge)
				// Save the event with the current time
				events[time.Now()] = recwatch.Event(ev)
				mut.Unlock()
				//log.Info(ev)
			case err := <-watcher.Errors:
				log.Error(err)
			}
		}
	}()
}

// Create events whenever a file in the server directory changes
func genFileChangeEvents(events TimeEventMap, mut *sync.Mutex, maxAge time.Duration, allowed string) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream;charset=utf-8")
		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("Connection", "keep-alive")
		w.Header().Set("Access-Control-Allow-Origin", allowed)

		var id uint64

		for {
			func() { // Use an anonymous function, just for using "defer"
				mut.Lock()
				defer mut.Unlock()
				if len(events) > 0 {
					// Remove old keys
					removeOldEvents(&events, maxAge)
					// Sort the events by the registered time
					var keys timeKeys
					for k := range events {
						keys = append(keys, k)
					}
					sort.Sort(keys)
					prevname := ""
					for _, k := range keys {
						ev := events[k]
						if verboseMode {
							log.Info("EVENT " + ev.String())
						}
						// Avoid sending several events for the same filename
						if ev.Name != prevname {
							// Send an event to the client
							Event(w, &id, ev.Name, true)
							id++
							prevname = ev.Name
						}
					}
				}
			}()
			// Wait for old events to be gone, and new to appear
			time.Sleep(maxAge)
		}
	}
}

// EventServer serves events on a dedicated port.
// addr is the host address ([host][:port])
// urlPath is the path to handle (ie /fs)
// refresh is how often the event buffer should be checked and cleared.
// The filesystem events are gathered independently of that.
// Allowed can be "*" or a hostname and sets a header in the SSE stream.
func EventServer(addr, urlPath, path string, refresh time.Duration, allowed string) {
	// Create a new filesystem watcher
	rw, err := recwatch.NewRecursiveWatcher(path)
	if err != nil {
		fatalExit(err)
	}

	var mut sync.Mutex
	events := make(TimeEventMap)

	// Collect the events for the last n seconds, repeatedly
	// Runs in the background
	collectFileChangeEvents(rw, &mut, events, refresh)

	if strings.Contains(addr, ":") {
		fields := strings.Split(addr, ":")
		log.Info("Serving filesystem events on port " + fields[1])
	}

	// Serve events
	go func() {
		eventMux := http.NewServeMux()
		// Fire off events whenever a file in the server directory changes
		eventMux.HandleFunc(urlPath, genFileChangeEvents(events, &mut, refresh, allowed))
		eventServer := &http.Server{
			Addr:    addr,
			Handler: eventMux,
			// ReadTimeout:  3600 * time.Second,
			// WriteTimeout: 3600 *	 time.Second,
			// MaxHeaderBytes: 1 << 20,
		}
		if err := eventServer.ListenAndServe(); err != nil {
			// If we can't serve HTTP on this port, give up
			fatalExit(err)
		}
	}()
}

// Insert JavaScript that refreshes the page when the source files changes.
// The JavaScript depends on the event server being available.
// If javascript can not be inserted, return the original data.
func insertAutoRefresh(htmldata []byte) []byte {
	fullHost := eventAddr
	// If the host+port starts with ":", assume it's only the port number
	if strings.HasPrefix(fullHost, ":") {
		// Add the hostname in front
		if serverHost != "" {
			fullHost = serverHost + eventAddr
		} else {
			fullHost = "localhost" + eventAddr
		}
	}
	// Wait 70% of an event duration before starting to listen for events
	multiplier := 0.7
	js := `
    `

	// Reduce the size slightly
	js = strings.TrimSpace(strings.Replace(js, "\n", "", everyInstance))
	// Remove all whitespace that is more than one space
	for strings.Contains(js, "  ") {
		js = strings.Replace(js, "  ", " ", everyInstance)
	}
	// Place the script at the end of the body, if there is a body
	if bytes.Contains(htmldata, []byte("")) {
		return bytes.Replace(htmldata, []byte(""), []byte(js+""), 1)
	} else if bytes.Contains(htmldata, []byte("")) {
		// If not, place the script in the , if there is a head
		return bytes.Replace(htmldata, []byte(""), []byte(""+js), 1)
	} else if bytes.Contains(htmldata, []byte("")) {
		// If not, place the script in the  as a new
		return bytes.Replace(htmldata, []byte(""), []byte(""+js+""), 1)
	}
	// In the unlikely event that no place to insert the JavaScript was found
	return htmldata
}
<|endoftext|>


<commit_before>// Azure VHD Utilities for Go
// Copyright (c) Microsoft Corporation
//
// All rights reserved.
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is furnished to do
// so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// derived from https://github.com/Microsoft/azure-vhd-utils/blob/8fcb4e03cb4c0f928aa835c21708182dbb23fc83/vhdUploadCmdHandler.go

package azure

import (
	"fmt"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Microsoft/azure-vhd-utils-for-go/upload"
	"github.com/Microsoft/azure-vhd-utils-for-go/upload/metadata"
	"github.com/Microsoft/azure-vhd-utils-for-go/vhdcore/common"
	"github.com/Microsoft/azure-vhd-utils-for-go/vhdcore/diskstream"
	"github.com/coreos/pkg/multierror"
)

const pageBlobPageSize int64 = 2 * 1024 * 1024

type BlobExistsError string

func (be BlobExistsError) Error() string {
	return fmt.Sprintf("blob %q already exists", string(be))
}

// UploadBlob uploads vhd to the given storage account, container, and blob name.
//
// It returns BlobExistsError if the blob exists and overwrite is not true.
func (a *API) UploadBlob(storageaccount, storagekey, vhd, container, blob string, overwrite bool) error {
	ds, err := diskstream.CreateNewDiskStream(vhd)
	if err != nil {
		return err
	}
	defer ds.Close()

	sc, err := storage.NewBasicClient(storageaccount, storagekey)
	if err != nil {
		return err
	}

	bsc := sc.GetBlobService()
	if _, err = bsc.CreateContainerIfNotExists(container, storage.ContainerAccessTypePrivate); err != nil {
		return err
	}

	blobExists, err := bsc.BlobExists(container, blob)
	if err != nil {
		return err
	}

	resume := false
	var blobMetaData *metadata.MetaData
	if blobExists {
		if !overwrite {
			bm, err := getBlobMetaData(bsc, container, blob)
			if err != nil {
				return err
			}
			blobMetaData = bm
			resume = true
			plog.Printf("Blob with name '%s' already exists, checking if upload can be resumed", blob)
		}
	}

	localMetaData, err := getLocalVHDMetaData(vhd)
	if err != nil {
		return err
	}
	var rangesToSkip []*common.IndexRange
	if resume {
		if errs := metadata.CompareMetaData(blobMetaData, localMetaData); len(errs) != 0 {
			return multierror.Error(errs)
		}
		ranges, err := getAlreadyUploadedBlobRanges(bsc, container, blob)
		if err != nil {
			return err
		}
		rangesToSkip = ranges
	} else {
		createBlob(bsc, container, blob, ds.GetSize(), localMetaData)
	}

	uploadableRanges, err := upload.LocateUploadableRanges(ds, rangesToSkip, pageBlobPageSize)
	if err != nil {
		return err
	}

	uploadableRanges, err = upload.DetectEmptyRanges(ds, uploadableRanges)
	if err != nil {
		return err
	}

	cxt := &upload.DiskUploadContext{
		VhdStream:             ds,
		UploadableRanges:      uploadableRanges,
		AlreadyProcessedBytes: common.TotalRangeLength(rangesToSkip),
		BlobServiceClient:     bsc,
		ContainerName:         container,
		BlobName:              blob,
		Parallelism:           8,
		Resume:                resume,
		MD5Hash:               localMetaData.FileMetaData.MD5Hash,
	}

	err = upload.Upload(cxt)
	if err != nil {
		return err
	}

	return nil
}

// getBlobMetaData returns the custom metadata associated with a page blob which is set by createBlob method.
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob resides, parameter blobName is name for the page blob
// This method attempt to fetch the metadata only if MD5Hash is not set for the page blob, this method panic if the
// MD5Hash is already set or if the custom metadata is absent.
//
func getBlobMetaData(client storage.BlobStorageClient, containerName, blobName string) (*metadata.MetaData, error) {
	md5Hash, err := getBlobMD5Hash(client, containerName, blobName)
	if md5Hash != "" {
		return nil, BlobExistsError(blobName)
	}

	blobMetaData, err := metadata.NewMetadataFromBlob(client, containerName, blobName)
	if err != nil {
		return nil, err
	}

	if blobMetaData == nil {
		return nil, fmt.Errorf("There is no upload metadata associated with the existing blob '%s', so upload operation cannot be resumed, use --overwrite option.", blobName)
	}

	return blobMetaData, nil
}

// getLocalVHDMetaData returns the metadata of a local VHD
//
func getLocalVHDMetaData(localVHDPath string) (*metadata.MetaData, error) {
	localMetaData, err := metadata.NewMetaDataFromLocalVHD(localVHDPath)
	if err != nil {
		return nil, err
	}
	return localMetaData, nil
}

// createBlob creates a page blob of specific size and sets custom metadata
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob needs to be created, parameter blobName is name for the new page blob, size is the size of
// the new page blob in bytes and parameter vhdMetaData is the custom metadata to be associacted with the page blob
//
func createBlob(client storage.BlobStorageClient, containerName, blobName string, size int64, vhdMetaData *metadata.MetaData) error {
	if err := client.PutPageBlob(containerName, blobName, size, nil); err != nil {
		return err
	}
	m, _ := vhdMetaData.ToMap()
	if err := client.SetBlobMetadata(containerName, blobName, m, make(map[string]string)); err != nil {
		return err
	}

	return nil
}

// getAlreadyUploadedBlobRanges returns the range slice containing ranges of a page blob those are already uploaded.
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob resides, parameter blobName is name for the page blob
//
func getAlreadyUploadedBlobRanges(client storage.BlobStorageClient, containerName, blobName string) ([]*common.IndexRange, error) {
	existingRanges, err := client.GetPageRanges(containerName, blobName)
	if err != nil {
		return nil, err
	}
	var rangesToSkip = make([]*common.IndexRange, len(existingRanges.PageList))
	for i, r := range existingRanges.PageList {
		rangesToSkip[i] = common.NewIndexRange(r.Start, r.End)
	}
	return rangesToSkip, nil
}

// getBlobMD5Hash returns the MD5Hash associated with a blob
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob resides, parameter blobName is name for the page blob
//
func getBlobMD5Hash(client storage.BlobStorageClient, containerName, blobName string) (string, error) {
	properties, err := client.GetBlobProperties(containerName, blobName)
	if err != nil {
		return "", err
	}
	return properties.ContentMD5, nil
}
<commit_msg>platform/api/azure: add BlobExists function<commit_after>// Azure VHD Utilities for Go
// Copyright (c) Microsoft Corporation
//
// All rights reserved.
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is furnished to do
// so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// derived from https://github.com/Microsoft/azure-vhd-utils/blob/8fcb4e03cb4c0f928aa835c21708182dbb23fc83/vhdUploadCmdHandler.go

package azure

import (
	"fmt"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Microsoft/azure-vhd-utils-for-go/upload"
	"github.com/Microsoft/azure-vhd-utils-for-go/upload/metadata"
	"github.com/Microsoft/azure-vhd-utils-for-go/vhdcore/common"
	"github.com/Microsoft/azure-vhd-utils-for-go/vhdcore/diskstream"
	"github.com/coreos/pkg/multierror"
)

const pageBlobPageSize int64 = 2 * 1024 * 1024

type BlobExistsError string

func (be BlobExistsError) Error() string {
	return fmt.Sprintf("blob %q already exists", string(be))
}

func (a *API) BlobExists(storageaccount, storagekey, container, blob string) (bool, error) {
	sc, err := storage.NewBasicClient(storageaccount, storagekey)
	if err != nil {
		return false, err
	}

	bsc := sc.GetBlobService()

	return bsc.BlobExists(container, blob)
}

// UploadBlob uploads vhd to the given storage account, container, and blob name.
//
// It returns BlobExistsError if the blob exists and overwrite is not true.
func (a *API) UploadBlob(storageaccount, storagekey, vhd, container, blob string, overwrite bool) error {
	ds, err := diskstream.CreateNewDiskStream(vhd)
	if err != nil {
		return err
	}
	defer ds.Close()

	sc, err := storage.NewBasicClient(storageaccount, storagekey)
	if err != nil {
		return err
	}

	bsc := sc.GetBlobService()
	if _, err = bsc.CreateContainerIfNotExists(container, storage.ContainerAccessTypePrivate); err != nil {
		return err
	}

	blobExists, err := bsc.BlobExists(container, blob)
	if err != nil {
		return err
	}

	resume := false
	var blobMetaData *metadata.MetaData
	if blobExists {
		if !overwrite {
			bm, err := getBlobMetaData(bsc, container, blob)
			if err != nil {
				return err
			}
			blobMetaData = bm
			resume = true
			plog.Printf("Blob with name '%s' already exists, checking if upload can be resumed", blob)
		}
	}

	localMetaData, err := getLocalVHDMetaData(vhd)
	if err != nil {
		return err
	}
	var rangesToSkip []*common.IndexRange
	if resume {
		if errs := metadata.CompareMetaData(blobMetaData, localMetaData); len(errs) != 0 {
			return multierror.Error(errs)
		}
		ranges, err := getAlreadyUploadedBlobRanges(bsc, container, blob)
		if err != nil {
			return err
		}
		rangesToSkip = ranges
	} else {
		createBlob(bsc, container, blob, ds.GetSize(), localMetaData)
	}

	uploadableRanges, err := upload.LocateUploadableRanges(ds, rangesToSkip, pageBlobPageSize)
	if err != nil {
		return err
	}

	uploadableRanges, err = upload.DetectEmptyRanges(ds, uploadableRanges)
	if err != nil {
		return err
	}

	cxt := &upload.DiskUploadContext{
		VhdStream:             ds,
		UploadableRanges:      uploadableRanges,
		AlreadyProcessedBytes: common.TotalRangeLength(rangesToSkip),
		BlobServiceClient:     bsc,
		ContainerName:         container,
		BlobName:              blob,
		Parallelism:           8,
		Resume:                resume,
		MD5Hash:               localMetaData.FileMetaData.MD5Hash,
	}

	err = upload.Upload(cxt)
	if err != nil {
		return err
	}

	return nil
}

// getBlobMetaData returns the custom metadata associated with a page blob which is set by createBlob method.
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob resides, parameter blobName is name for the page blob
// This method attempt to fetch the metadata only if MD5Hash is not set for the page blob, this method panic if the
// MD5Hash is already set or if the custom metadata is absent.
//
func getBlobMetaData(client storage.BlobStorageClient, containerName, blobName string) (*metadata.MetaData, error) {
	md5Hash, err := getBlobMD5Hash(client, containerName, blobName)
	if md5Hash != "" {
		return nil, BlobExistsError(blobName)
	}

	blobMetaData, err := metadata.NewMetadataFromBlob(client, containerName, blobName)
	if err != nil {
		return nil, err
	}

	if blobMetaData == nil {
		return nil, fmt.Errorf("There is no upload metadata associated with the existing blob '%s', so upload operation cannot be resumed, use --overwrite option.", blobName)
	}

	return blobMetaData, nil
}

// getLocalVHDMetaData returns the metadata of a local VHD
//
func getLocalVHDMetaData(localVHDPath string) (*metadata.MetaData, error) {
	localMetaData, err := metadata.NewMetaDataFromLocalVHD(localVHDPath)
	if err != nil {
		return nil, err
	}
	return localMetaData, nil
}

// createBlob creates a page blob of specific size and sets custom metadata
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob needs to be created, parameter blobName is name for the new page blob, size is the size of
// the new page blob in bytes and parameter vhdMetaData is the custom metadata to be associacted with the page blob
//
func createBlob(client storage.BlobStorageClient, containerName, blobName string, size int64, vhdMetaData *metadata.MetaData) error {
	if err := client.PutPageBlob(containerName, blobName, size, nil); err != nil {
		return err
	}
	m, _ := vhdMetaData.ToMap()
	if err := client.SetBlobMetadata(containerName, blobName, m, make(map[string]string)); err != nil {
		return err
	}

	return nil
}

// getAlreadyUploadedBlobRanges returns the range slice containing ranges of a page blob those are already uploaded.
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob resides, parameter blobName is name for the page blob
//
func getAlreadyUploadedBlobRanges(client storage.BlobStorageClient, containerName, blobName string) ([]*common.IndexRange, error) {
	existingRanges, err := client.GetPageRanges(containerName, blobName)
	if err != nil {
		return nil, err
	}
	var rangesToSkip = make([]*common.IndexRange, len(existingRanges.PageList))
	for i, r := range existingRanges.PageList {
		rangesToSkip[i] = common.NewIndexRange(r.Start, r.End)
	}
	return rangesToSkip, nil
}

// getBlobMD5Hash returns the MD5Hash associated with a blob
// The parameter client is the Azure blob service client, parameter containerName is the name of an existing container
// in which the page blob resides, parameter blobName is name for the page blob
//
func getBlobMD5Hash(client storage.BlobStorageClient, containerName, blobName string) (string, error) {
	properties, err := client.GetBlobProperties(containerName, blobName)
	if err != nil {
		return "", err
	}
	return properties.ContentMD5, nil
}
<|endoftext|>