# November, 2020
# Kevin Dorma
# module for common hydraulic calculations
# rev 0, December 2020
# rev 1, February 2021, added network hydraulics

# I need to make some decisions about how to interace with these functions
# I assume that I use dataframes to store information about sizing lines and general hydraulics
# then I will execute a function and it will return a result, but not change the raw information
# size lines
    # Input dataframe for describing the lines
    # Output line descriptor, pipe schedule, size criteria DP/100 and C, IDmm for each criteria and the chosen IDmm
# pick NPS will return the next size larger for NPS.This will be the chosen size.
# we will have utility functions for increasing or decreasing the NPS by 1.

module Hydraulics2

using DataFrames
using CSV
using ExcelFiles # more convenient than CSV for user input

export calcMoodyF
export addPipeProperties, addFluidProperties, getReynolds, getSegmentDP
export getLineSize, getListLargerNPS, getLargerNPS
# and the network hydraulic files
export indexLookup, doNetworkHydraulics, extractResults, combineResults

# these are the reference data.
# and this needs to be loaded correctly when we use the code in a module
include("npsList.jl")
include("schedList.jl")
include("pipeRoughnessList.jl")
include("fitting3K.jl")
include("pipeIDlist.jl")

#schedList = CSV.File("schedList.csv") |> DataFrame
#pipeRoughness = CSV.File("pipeRoughness.csv") |> DataFrame
#fitting3K = CSV.File("fitting3K.csv") |> DataFrame
#IDmm = CSV.File("pipeIDlist.csv") |> DataFrame
## not used
#pipeTable = CSV.read("pipeIDtable.csv");

function packageInfo()
    # return information about the package as a string
    return("Hydraulics package, Kevin Dorma. Written in Julia, rev 1 February 2021.")
end

# need function to append piping data
# later

function calcMoodyF(Reynolds,eD)
    # Moody friction factor, this is Halaand
    # eD is e/D relative roughness
    # Reynolds is Reynolds number
    invSqrtF = -1.8 .* log10.((eD ./ 3.70) .^ 1.11 + 6.9 ./ Reynolds)
    moodyF = (1 ./ (invSqrtF .^2) )
    return (moodyF)
end

function checkFittingList(fittingList)
    # fittingList is our list of fittings, fittingReference is the generic data
    # go through the list of fittings (fittingList) and compare with the reference list (possibly our fitting3K global list)
    # return items that are not found
    errorList = DataFrame(message = String[], entry=Int64[], Segment=String[], fittingType=String[])
    for i = 1:(size(fittingList)[1])
        thisFitting = fittingList[i,:fittingType]
        match = fitting3K[fitting3K.fittingType .== thisFitting,:]
        if ((size(match)[1]) == 0)
            push!(errorList, ("Fitting not found in row", i, fittingList[i,:Segment], fittingList[i,:fittingType] ))
        end
    end
    return (errorList)
end

function checkLineList(lines,fluidList)
    # go through our list of lines and compare with our reference lists
    # return the line items that are not found
    errorList = DataFrame(message = String[], entry=Int64[], Segment=String[], NPS=Float64[], Schedule=String[], material=String[], fluidName=String[])
    for i = 1:(size(lines)[1])
        # NPS, must check if this field exists
	if (indexLookup("NPS",names(lines)) > 0)
        thisItem = lines[i,:NPS]
        match = npsList[npsList.NPS .== thisItem,:]
        if ((size(match)[1]) == 0)
            push!(errorList, ("NPS not found in row", i, lines[i,:Segment], lines[i,:NPS], lines[i,:Schedule], lines[i,:Material], lines[i,:fluidName] ));
        end;
	end;
        # schedule
        thisItem = lines[i,:Schedule]
        match = schedList[schedList.Schedule .== thisItem,:]
        if ((size(match)[1]) == 0)
            push!(errorList, ("Schedule not found in row", i, lines[i,:Segment], lines[i,:NPS], lines[i,:Schedule], lines[i,:Material], lines[i,:fluidName] ))
        end
        # roughness material
        thisItem = lines[i,:Material]
        match = pipeRoughness[pipeRoughness.Material .== thisItem,:]
        if ((size(match)[1]) == 0)
            push!(errorList, ("Material not found in row", i, lines[i,:Segment], lines[i,:NPS], lines[i,:Schedule], lines[i,:Material], lines[i,:fluidName] ))
        end
        # fluid
        thisItem = lines[i,:fluidName]
        match = fluidList[fluidList.fluidName .== thisItem,:]
        if ((size(match)[1]) == 0)
            push!(errorList, ("FluidName not found in row", i, lines[i,:Segment], lines[i,:NPS], lines[i,:Schedule], lines[i,:Material], lines[i,:fluidName] ))
        end
    end
    return (errorList)
end

function addPipeProperties(df)
    # roughness, IDmm, eD ratio
    # this appends columns to df
    # used either line sizing or hydraulic calculations

    df[:,:roughnessMM] .= 0.0
    df[:,:IDmm] .= 0.0

    for i = 1:(size(df)[1])
        df[i,:roughnessMM]=pipeRoughness[df[i,:Material] .== pipeRoughness[:,:Material],:roughnessMM][1]
        df[i,:IDmm]= IDmm[(df[i,:NPS] .== IDmm[:,:NPS]) .& (df[i,:Schedule] .== IDmm[:,:Schedule]),:Idmm][1]
    end

    return (0.0)
end

function addFluidProperties(df,fluidList)
    # extract the density, viscosity and whatever else is needed from the fluidList
    # Add this to the df
    # used either line sizing or hydraulic calculations

    df[:,:rho_kgm3] .= 0.0
    df[:,:mu_mPas] .= 0.0
    df[:,:roughnessMM] .= 0.0

    for i = 1:(size(df)[1])
        df[i,:rho_kgm3] = fluidList[df[i,:fluidName] .== fluidList[:,:fluidName], :rho_kgm3][1]
        df[i,:mu_mPas]  = fluidList[df[i,:fluidName] .== fluidList[:,:fluidName], :mu_mPas][1]
        df[i,:roughnessMM]  = pipeRoughness[df[i,:Material] .== pipeRoughness[:,:Material], :roughnessMM][1]
    end

    return (0.0)
end

function lineSizeDP(df)
    # this is for sizing lines based on pressure drop kPa per 100 m, return the ID in mm
    # very convenient form straight out of Perrys handbook
    # uses friction factor correlation from Chen
    # Do not modify the original data in df

    g = 9.80665
    Sf = df.kPaPer100m*1000 ./ (df.rho_kgm3*g*100)
    q = df.massFlow .* df.margin ./(df.rho_kgm3*3600)
    eps = (df.roughnessMM/1000)
    kinVisc = (df.mu_mPas/1000) ./ df.rho_kgm3
    termA = ((eps.^5)*g .* Sf ./ (q.^2)).^0.25
    termB = ((kinVisc .^ 5)./((q.^3) .* Sf * g )).^0.2
    termC = 0.125*(termA .+ termB).^0.2
    returnValue = (1000*(termC .* (q.^2) ./ (g*Sf)).^0.2)
    return (returnValue)
end

function lineSizeErosion(df)
    # df is the dataframe with hydraulic information
    # this is the erosion C factor method
    # C = v/sqrt(rho), with v in m/s and rho in kg/m3
    # for most service, C = 120, this is similar to C = 100 for imperial units
    # return the ID in mm

    g = 9.80665
    maxVeloc = df.frictionCsi ./ sqrt.(df.rho_kgm3)
    q = df.massFlow .* df.margin ./(df.rho_kgm3*3600)
    area = q ./ maxVeloc
    returnValue = (sqrt.(4*area ./ pi )*1000.0)
    return (returnValue)
end

function getLineSize(df,fluidList)
    # from the calculated line size for the two different methods, determine the required line size

    addFluidProperties(df,fluidList)

    theSchedule = df[:,:Schedule]

    theSegment = df[:,:Segment]

    tempDP100 = lineSizeDP(df)
    tempErosion = lineSizeErosion(df)
    tempNeeded = tempErosion[:] + tempDP100[:]
    for j in 1:size(tempDP100)[1]
        tempNeeded[j] =max(tempDP100[j], tempErosion[j])
    end
    returnValue = DataFrame(Segment = theSegment, mmDP100 = tempDP100, mmErosion = tempErosion, mmNeeded = tempNeeded, Schedule = theSchedule)
    return (returnValue)
end

# I need a function to pick the next larger line size given the pipe schedule

function getLargerNPS(ourIDmm, ourSchedule)
    # from the common list of line sizes, pick the pipe size that matches the required schedule
    # and is the next larger that the required ID
    # this works for a single line size
    ourScheduleList = IDmm[IDmm[:,:Schedule] .== ourSchedule, :];
    refinedList = ourScheduleList[(ourScheduleList[:,:Idmm] .- ourIDmm) .> 0.0,:];
    theMinID = minimum(refinedList[:,:Idmm])
    ourNPS = refinedList[(refinedList[:,:Idmm] .== theMinID), :NPS]

    return (ourNPS)
end

function getListLargerNPS(lineSizeDF)
    # we will use the required line size and schedule and return the NPS and actual ID
    diamList = DataFrame(Segment=String[], NPS=Float64[], Schedule=String[], IDmm=Float64[])
    for i = 1:(size(lineSizeDF)[1])
        ourSchedule = lineSizeDF[i,:Schedule];
        ourIDmm = lineSizeDF[i,:mmNeeded]
        ourScheduleList = IDmm[IDmm[:,:Schedule] .== ourSchedule, :];
        refinedList = ourScheduleList[(ourScheduleList[:,:Idmm] .- ourIDmm) .> 0.0,:];
        theMinID = minimum(refinedList[:,:Idmm])
        ourNPS = refinedList[(refinedList[:,:Idmm] .== theMinID), :NPS][1]

        push!(diamList, (lineSizeDF[i,:Segment], ourNPS, ourSchedule, theMinID))
    end
    return (diamList)
end

function sizing2hydraulics(sizingDF, chosenLineSizeDF)
    # given the line sizing DF, copy the info into hydraulics dataframe
    # chosenSizeDF contains the NPS and Schedule
    # return the hydraulics DF
    # we also need a simple fittingDF, but this is done with a separate function
    hydraulicsDF = DataFrame(Segment=String[], Description=String[], LineTag=String[], PnID=String[], NPS=Float64[], Schedule=String[],
		Material=String[],	fluidName=[],	inletP_kPaa=[],	massFlow=Float64[])
    for i = 1:(size(sizingDF)[1])
        push!(hydraulicsDF, (sizingDF[i,:Segment], sizingDF[i,:Description],
                sizingDF[i,:LineTag], sizingDF[i,:PnID], chosenLineSizeDF[i,:NPS],
                chosenLineSizeDF[i,:Schedule], sizingDF[i,:Material], sizingDF[i,:fluidName],
                100.0, (sizingDF[i,:massFlow]*sizingDF[i,:margin])))
    end
    return(hydraulicsDF)
end

function sizing2fittingList(sizingDF, chosenLineSizeDF)
    # given the line sizing DF, copy the info into the fitting list
    # chosenSizeDF contains the NPS and Schedule
    # return the pre-populsted fittign list DF
    fittingListDF = DataFrame(Segment=String[], fittingType=String[], num_length_m=Float64[], comment=String[], revision=String[])

    for i = 1:(size(sizingDF)[1])
        push!(fittingListDF, (sizingDF[i,:Segment], "PIPE", 100.0, "from line sizing", "A"))
    end
    return(fittingListDF)
end

function incrementLineSize(rowNum, lineSizeDF, increment)
    # increment the specified row number needed line size by +1 or -1
    # not the cleanest function, but it works

    neededDF = DataFrame(Segment=String[], mmNeeded=Float64[], Schedule=String[])

    i = rowNum
    push!(neededDF, (lineSizeDF[i,:Segment], lineSizeDF[i,:IDmm] + increment, lineSizeDF[i,:Schedule]))

    incrementedLines = getListLargerNPS(neededDF)

    lineSizeDF[i,:NPS] = incrementedLines[1,:NPS]
    lineSizeDF[i,:IDmm] = incrementedLines[1,:IDmm]

    return (3.14)
end

function getReynolds(df)
    # preliminary calculations for pipe hydraulics
    # return the velocity, Reynolds, friction factor, density, diameterMM, diameterInch

    theSegment = df.Segment
    diamMM = df.IDmm
    diamInch = diamMM / 25.4
    rho = df.rho_kgm3
    volFlow = df.massFlow ./ df.rho_kgm3
    area = 0.25 * pi * (df.IDmm / 1000.0).^2
    velocity = volFlow ./ area / 3600
    Reynolds = df.rho_kgm3 .* velocity .* (df.IDmm / 1000.0) ./ (df.mu_mPas/1000.0)
    eD = df.roughnessMM ./ df.IDmm
    moodyF = calcMoodyF(Reynolds, eD)
    returnValue = DataFrame(Segment = theSegment, velocity_ms = velocity, rho_kgm3 = rho, Re = Reynolds, frictF = moodyF, IDmm = diamMM, IDinch = diamInch)
    return (returnValue)
end

function elementDP(lines, fittingList)
    # given the hydraulic elements in df (long list of pipe and fittings)
    # and the preliminary values in prelim
    # calculate the DP in each element
    # dp = rho.f. (L/D) v2/2
    # dp = rho K v2/2
    # where I need to calculate K = K1/Re + Kinf*(1 + Kd/Dinch^0.3)
    # can I get all of the pipe segments and use K = Kp * f L/D, where Kp = 1 for pipe
    # return the value for Kp (1 for pipe, 0 for fitting) and the pressure drop in the item
    returnValue = DataFrame(Segment = String[], Kp = Float64[], fittingK=Float64[], pipeK=Float64[], elementK = Float64[], DPkpa = Float64[])

    # start with getting Reynolds and other important things
    prelim = getReynolds(lines)

    for i = 1:(size(fittingList)[1])
        theSegment = fittingList[i,:Segment]

        # first we get all of the K values
        valK1 = fitting3K[fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:K1][1]
        valKinf = fitting3K[fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:Kinf][1]
        valKd = fitting3K[fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:Kd][1]
        valKp = fitting3K[fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:Kp][1]

        # now we get the hydraulic properties
        ff = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:frictF][1]
        rho = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:rho_kgm3][1]
        idInch = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:IDinch][1]
        idMM = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:IDmm][1]
        veloc = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:velocity_ms][1]
        Re = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:Re][1]

        Kfitting = valK1/Re + valKinf*(1.0 + valKd/(idInch^0.3))

        # pressure drop in kPa
        thisKfitting = fittingList[i,:num_length_m] * Kfitting
        thisKpipe = valKp * ff * fittingList[i,:num_length_m] /(idMM/1000.0)
        thisK = thisKfitting + thisKpipe
        thisDP = thisK * (0.001 * rho * 0.5 * (veloc)^ 2)

        push!(returnValue, (theSegment, valKp, thisKfitting, thisKpipe, thisK, thisDP))
    end
    return (returnValue)
end

function compileDP(lines, fittingDP)
    # for each entry in lines, sum all of the fittingDP
    returnValue = DataFrame(Segment = String[], segmentK = Float64[], DPkpa = Float64[], inletP = Float64[], outletP = Float64[])

    for i = 1:(size(lines)[1])
        theSegment = lines[i,:Segment]
        theSumK = sum(fittingDP[fittingDP[:,:Segment] .== theSegment,:elementK])
        theSumDP = sum(fittingDP[fittingDP[:,:Segment] .== theSegment,:DPkpa])
        theInletP = lines[i,:inletP_kPaa]
        theOutletP = theInletP - theSumDP
        push!(returnValue, (theSegment, theSumK, theSumDP, theInletP, theOutletP))
    end
    return (returnValue)
end

function getSegmentDP(lines, fittingList, fluidList)
    # lines describes the hydraulics, fittingList has the piping details, fluidList is the fluid

    addFluidProperties(lines,fluidList)
    addPipeProperties(lines)
    fittingDP = elementDP(lines, fittingList)
    returnValue = compileDP(lines, fittingDP)
    return (returnValue)
end

# functions added for network hydraulics
# february, 2021
# exported
function indexLookup(findMe, inThis)
    # find the index where findMe can be found inThis, which is a vector
    maxI = size(inThis)[1]
    foundIdx = 0
    for i = 1:maxI
        if (findMe == inThis[i])
            foundIdx = i
        end
    end
    return(foundIdx)
end

function getConnectivity(connectivity, nodeList)
    # connectivity, lineList and nodeList are dataframe
#    maxLine = size(lineList)[1]

    maxLine = size(connectivity)[1]

    maxNode = size(nodeList)[1]
    C = zeros(Float64,maxLine,maxNode)

    maxConnect = size(connectivity)[1]

    for i = 1:maxConnect
        theSegment = connectivity[i,:Segment]
        inNode = connectivity[i,:inNode]
        outNode = connectivity[i,:outNode]
#        theRow = indexLookup(theSegment,lineList[:,:Segment])
        theRow = indexLookup(theSegment,connectivity[:,:Segment])
        inCol = indexLookup(inNode,nodeList[:,:Node])
        outCol = indexLookup(outNode,nodeList[:,:Node])
        C[theRow,inCol] = 1.0
        C[theRow,outCol] = -1.0
    end
    return(C)
end

function removeTrivialRows(A)
    # remove rows from matrix A that contain only one entry
    numRows = size(A)[1]
    numColumns = size(A)[2]
    indexList = zeros(Int64, numRows,1)
    numNonTrivialRows = 0
    for i = 1:numRows
        countNonZero = 0
        for j = 1:numColumns
            if (A[i,j] != 0.0)
                countNonZero = countNonZero + 1
            end
        end
        if (countNonZero >= 2)
            numNonTrivialRows += 1
            indexList[numNonTrivialRows] = i
        end

    end
    returnMatrix = zeros(numNonTrivialRows,numColumns)
    for i = 1:numNonTrivialRows
        for j = 1:numColumns
            returnMatrix[i,j] = A[indexList[i],j]
        end
    end

    return(returnMatrix)
end

function initializeHydraulicParameters(setRe, df)
    # set all of the Reynolds numbers in lines df
    # return the hydraulicParameters (Reynolds, friction factor, density, diameterMM, diameterInch)

    theSegment = df.Segment
    diamMM = df.IDmm
    rho_kgm3 = df.rho_kgm3
    diamInch = diamMM / 25.4

    volFlow = df.massFlow ./ df.rho_kgm3
    area = 0.25 * pi * (df.IDmm / 1000.0).^2
    velocity = volFlow ./ area / 3600
    Reynolds = setRe .* (df.rho_kgm3 ./ df.rho_kgm3)
    eD = df.roughnessMM ./ df.IDmm
    moodyF = calcMoodyF(Reynolds, eD)

    returnValue = DataFrame(Segment = theSegment, rho_kgm3 = rho_kgm3, Re = Reynolds, frictF = moodyF, IDmm = diamMM, IDinch = diamInch)
    return (returnValue)
end

function initializeMassFlow(df, connectivity)
    # set mass flow in df to 1000 kg/h
    # but only if the flow rate is undefined or zero
    # I want to create a df with fields Segment and massFlow
    # we have viscosity, density and IDmm in the df
    # we will use a frictional pressure gradient of 20 kPa per 100 m for initial guess on mass flow
    # and friction factor 0.02
    # but then overwrite if we have something filled in for massFlow in the df

    returnValue = DataFrame(Segment = String[], massFlow = Float64[])

    # first, set all mass flows equal to 1000 kg/h
    numRows = size(connectivity)[1]
    for i = 1:numRows
        push!(returnValue, (connectivity[i,:Segment], 1000.0) )
    end

    # now, plow thorugh the hydraulic information
    # if there is no information here, then use DP per 100m to calculate a mass flow
    # if there is information here, use this
    numHydraulicRows = size(df)[1]

    dp100 = 20.0
    ff = 0.02 # estimate for friction factor

    for i = 1:numHydraulicRows
        ii = indexLookup(df[i,:Segment],connectivity[:,:Segment])
        # calculate this just in case
        dd = df[i,:IDmm]
        rho = df[i,:rho_kgm3]
        visc = df[i,:mu_mPas]
        mm = 3600 * sqrt(20*1000*rho*(pi^2 / (8*ff))*((dd^5)/100)*(1/1000)^5)
        if (ismissing(df[i,:massFlow]) || (df[i,:massFlow] == 0.0) )
            df[i,:massFlow] = mm
            returnValue[ii,:massFlow] = mm
        else
            returnValue[ii,:massFlow] = df[i,:massFlow]
        end
    end
    return (returnValue)
end

function getResistanceMatrix(hydraulicParameters, elementK, connectivity)
    # given the hydraulic parameters, a list of all segments
    # and the long list of K values for every element in the network
    # calculate the sum of the overall K for each segment
    # return a diagonal matrix with the K values on the diagonal
    # wait, the resistance K value is in terms of velocity in m/s
    # I need mass flow in kg/h
    #
    # I think I need to add the connectivity matrix to get the correct value for the variable number

    numRows = size(hydraulicParameters)[1]
    numColumns = size(connectivity)[1]

    numElements = size(elementK)[1]
#    matrixK = zeros(Float64,numRows,numRows)
#     this needs to be the number of variables in the connectivity matrix
    matrixK = zeros(Float64,numRows,numColumns)

    for i = 1:numElements
#        theRow = indexLookup(elementK[i,:Segment], hydraulicParameters[:,:Segment])
#        theRow = indexLookup(elementK[i,:Segment], hydraulicParameters[:,:Segment])

        theRow = indexLookup(elementK[i,:Segment], hydraulicParameters[:,:Segment])
        theColumn = indexLookup(elementK[i,:Segment], connectivity[:,:Segment])

        matrixK[theRow,theColumn] += elementK[i,:elementK]
    end
    # now we multiply each row element by 8 x 1000 / (rho pi^2 D^4) (1000/3600)^2
    # where D is in mm, mass flow in kg/h and DP in kPa
    for i = 1:numRows
        multK = 8.0*1000.0 * ((1000/3600)^2) / ((pi^2) * hydraulicParameters[i,:rho_kgm3] * (hydraulicParameters[i,:IDmm]^4)   )
        for j = 1:numColumns
            matrixK[i,j] *= multK
        end
    end
    return (matrixK)
end

function getSegLength(connectivity, fittingList)

    returnValue = DataFrame(Segment = String[], pipeLength=Float64[])

# initialize the segLength data
    for i=1:(size(connectivity)[1])
	push!(returnValue, (connectivity[i,:Segment], 0.0))
    end
    for i = 1:(size(fittingList)[1])
        valKp = fitting3K[fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:Kp][1]
	ii = indexLookup(fittingList[i,:Segment], connectivity[:,:Segment])
	if (valKp == 1)
		returnValue[ii,:pipeLength] += fittingList[i,:num_length_m]
	end
    end
    return(returnValue)
end

function getConstraintMatrix(constraintDF, constraintType, segmentDF, nodeDF)
    # given the constraint data, the type (massFlow or pressure or value), the list of segments and nodes
    # create a matrix with values of 1 in the row, or the actual value

    numConstraints = size(constraintDF)[1]
    numSegments = size(segmentDF)[1]
    numNodes = size(nodeDF)[1]

    # we need the dimensions for the return matrix
    # assume a value for now
    numCols = 2
    if (constraintType == "flowRate")
        numCols = numSegments
    end
    if (constraintType == "pressure")
        numCols = numNodes
    end
    if (constraintType == "value")
        numCols = 1
    end
    returnMatrix = zeros(numConstraints, numCols)

    # cycle through all of the constraints
    # of the constraintType matches the :variable in the constraintDF, then place a 1 in the matrix entry
    for i = 1:numConstraints
        if (constraintType == "value")
            returnMatrix[i] = constraintDF[i,:value]
        elseif (constraintType == "flowRate")
            varIndexLocal = indexLookup(constraintDF[i,:object], segmentDF[:,:Segment])
            if (varIndexLocal > 0)
                returnMatrix[i,varIndexLocal] = 1.0
            end
        elseif (constraintType == "pressure")
            varIndexLocal = indexLookup(constraintDF[i,:object], nodeDF[:,:Node])
            if (varIndexLocal > 0)
                returnMatrix[i,varIndexLocal] = 1.0
            end
        end
    end

    return(returnMatrix)
end

function getQuadraticMatrix(hydraulicDF, which)
    # given the estimates of mass flow rate in the hydraulic matrix, fill out the
    # linearized estimate to the quadratic mass flow^2
    # m^2 ~= 2 M m - M*M,
    # or
    # m2 - 2 M m = - M*M, where we are looking for m2 (m^2) and m, and M is the
    # previous estiamte of m
    # the value of which is either quadratic, linear, value

    # given the constraint data, the type (massFlow or pressure or value), the list of segments and nodes
    # create a matrix with values of 1 in the row, or the actual value

    numEquations = size(hydraulicDF)[1]
    numCols = numEquations

    if (which == "rhs")
        numCols = 1
    end

    returnMatrix = zeros(numEquations, numCols)

    # cycle through all of the segments
    # fill in the value for the matrix equation
    # m2 - 2 M m = - M*M, where we are looking for m2 (m^2) and m, and M is the
    # if quadratic rM[i,i] = 1.0
    # if linear rm[i,i] = -2*hydraulicDF[i,:massFlow]
    # if value rm[i] = -hydraulicDF[i,:massFlow]*hydraulicDF[i,:massFlow]

    for i = 1:numEquations
        if (which == "rhs")
            returnMatrix[i] = -hydraulicDF[i,:massFlow]*hydraulicDF[i,:massFlow]
        elseif (which == "quadratic")
            returnMatrix[i,i] = 1.0
        elseif (which == "linear")
            returnMatrix[i,i] = -2.0 * hydraulicDF[i,:massFlow]
        end
    end

    return(returnMatrix)
end

function getSegmentIMatrix(hydraulicParameters, connectivity)
    # given the Identity matrix for the segments
    # return Identity matrix

    numRows = size(hydraulicParameters)[1]
    numColumns = size(connectivity)[1]

    matrixI = zeros(Float64,numRows,numColumns)

    for i = 1:numRows
        findMe = hydraulicParameters[i,:Segment]
        inThis = connectivity[:,:Segment]
        j = indexLookup(findMe, inThis)
        matrixI[i,j] = 1.0
    end

    return (matrixI)
end

function getElementK(prelim, fittingList)
    # given the hydraulic elements in df (long list of pipe and fittings)
    # and the preliminary friction properties (Re, eD, ff)
    # calculate the loss coefficient K in each element
    # where I need to calculate K = K1/Re + Kinf*(1 + Kd/Dinch^0.3)
    # can I get all of the pipe segments and use K = Kp * f L/D, where Kp = 1 for pipe
    # return the value for Kp (1 for pipe, 0 for fitting) and the pressure drop in the item
    returnValue = DataFrame(Segment = String[], Kp = Float64[], fittingK=Float64[], pipeK=Float64[], elementK = Float64[], pipeLength=Float64[])

    for i = 1:(size(fittingList)[1])
        theSegment = fittingList[i,:Segment]

        # first we get all of the K values
        valK1 = Hydraulics2.fitting3K[Hydraulics2.fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:K1][1]
        valKinf = Hydraulics2.fitting3K[Hydraulics2.fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:Kinf][1]
        valKd = Hydraulics2.fitting3K[Hydraulics2.fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:Kd][1]
        valKp = Hydraulics2.fitting3K[Hydraulics2.fitting3K[:,:fittingType] .== fittingList[i,:fittingType],:Kp][1]

        # now we get the hydraulic properties
        ff = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:frictF][1]
        rho = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:rho_kgm3][1]
        idInch = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:IDinch][1]
        idMM = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:IDmm][1]
#        veloc = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:velocity_ms][1]
        Re = prelim[prelim[:,:Segment] .== fittingList[i,:Segment],:Re][1]

        Kfitting = valK1/Re + valKinf*(1.0 + valKd/(idInch^0.3))

        # pressure drop in kPa
        thisKfitting = fittingList[i,:num_length_m] * Kfitting
        thisKpipe = valKp * ff * fittingList[i,:num_length_m] /(idMM/1000.0)
	if (valKp == 1)
		thisPipeLength = fittingList[i,:num_length_m]
	else
		thisPipeLength = 0.0
	end

        thisK = thisKfitting + thisKpipe
#        thisDP = thisK * (0.001 * rho * 0.5 * (veloc)^ 2)

        push!(returnValue, (theSegment, valKp, thisKfitting, thisKpipe, thisK, thisPipeLength))
    end
    return (returnValue)
end

function addToBlockMatrix(gBM, localMatrix, gR, gC, eqnType, varType, numLocalRows, numLocalColumns)
    # we will traverse the localMatrix and place the entries in the global block matrix
    # gR and gC are the table of indices that map gR[eqnType,i] to the row in global matrix
    # and gC[varType,j] to the column in the global matrix
    # eqnType and varType refer to which block
    # numLocalRows and Columns tells us how many local entries are avalable

    for i = 1:numLocalRows
        for j = 1:numLocalColumns
            gi = gR[eqnType,i]
            gj = gC[varType,j]
            gBM[gi,gj] = localMatrix[i,j]
        end
    end
    return(3.14)
end

function addToBlockRHS(gBv, localVec, gR, eqnType, numLocalRows)
    # we will traverse the localMatrix and place the entries in the global block matrix
    # gR and gC are the table of indices that map gR[eqnType,i] to the row in global matrix
    # and gC[varType,j] to the column in the global matrix
    # eqnType and varType refer to which block
    # numLocalRows and Columns tells us how many local entries are avalable

    for i = 1:numLocalRows
        gi = gR[eqnType,i]
        gBv[gi] = localVec[i]
    end
    return(3.14)
end

# create routines to extract results
# extractFromBlockSolution(testSolution, globalColumn, j (which type is it),columnsInSegment[j])
function extractFromBlockSoln(gBx, gR, varType, numLocalRows)
    # we will traverse the appropriate entries in the global block matrix
    # gR is the vector of indices that map gR[varType,i] to the row in global solution vector
    # varType refer to which block
    # numLocalRows tells us how many local entries are avalable

    localX=zeros(numLocalRows,1)
    for i = 1:numLocalRows
        gi = gR[varType,i]
        localX[i] = gBx[gi]
    end
    return(localX)
end

# create routines to extract results
# extractFromBlockSolution(testSolution, globalColumn, j (which type is it),columnsInSegment[j])
function catIt(dfHydraulic1, dfHydraulic2, dfAll)
    # dfAll has all of the rows, we need to fill in blank rows for the other dfs

    localX=zeros(numLocalRows,1)
    for i = 1:numLocalRows
        gi = gR[varType,i]
        localX[i] = gBx[gi]
    end
    return(localX)
end

# create routines to extract results
function extractResults(gBx, lines, connectivity, nodeList, fittingList)
    # we will traverse the solutions in the block x vector
    # lines is the df with the segments defined
    # connectivity is the connectivity df
    # we will find all of the from entries in the connectivity matrix
    numLines = getSize(connectivity)
    numHydraulicLines = getSize(connectivity)
    numNodes = getSize(nodeList)

    columnsInSegment = getColumnsInSegment(numLines,numNodes)
    globalColumn = variableMapping(numLines, numNodes, columnsInSegment)

    ourDetails = (Hydraulics2.getReynolds(lines))
	# note that fittingList is a really long vector with every fitting
    ourElementK = getElementK(ourDetails, fittingList)
    Km = getResistanceMatrix(ourDetails, ourElementK, connectivity)  # this is an actual matrix
    segLength = getSegLength(connectivity,fittingList)  # dataframe for length and other stuff

    segList = copy(connectivity[:,:Segment])
    nList = copy(nodeList[:,:Node])

    # connectivity, used for pressure continuity
    localA = getConnectivity(connectivity,nodeList)

    # extract all of the entries with +1 and -1
    fromA = max.(localA, 0)
    toA = -min.(localA, 0)

    varType = 1        # mass flow
    xMassFlow = extractFromBlockSoln(gBx, globalColumn, varType, columnsInSegment[varType])

    varType = 2     # mass flow^2
    xMassFlow2 = extractFromBlockSoln(gBx,globalColumn,varType,columnsInSegment[varType])

    varType = 3     # DP
    xDP = extractFromBlockSoln(gBx,globalColumn,varType,columnsInSegment[varType])

    varType = 4     # node pressure
    xPnode = extractFromBlockSoln(gBx,globalColumn,varType,columnsInSegment[varType])
    fromP = fromA*xPnode
    toP = toA*xPnode

    # messy, but it works
#    allResults = [segList xMassFlow xDP fromP toP Konly]
    allResults = [segList xMassFlow xDP fromP toP]

#    returnVal = DataFrame(Segment=allResults[:,1], massFlow=allResults[:,2], dp=allResults[:,3], fromP=allResults[:,4], toP=allResults[:,5], elementK=allResults[:,6])

    returnVal = DataFrame(Segment=allResults[:,1], massFlow=allResults[:,2], dp=allResults[:,3], fromP=allResults[:,4], toP=allResults[:,5], fromNode=connectivity[:,:inNode], toNode=connectivity[:,:outNode], segmentLength=segLength[:,:pipeLength])

    return(returnVal)
end

function getSize(df)
    # get the number of rows
    return(size(df)[1])
end

function getColumnsInSegment(numLines, numNodes)
    # define the number of variables in each of the different types of variables
    numEqnTypes = 5 # not used
    numVarTypes = 4 # m, m2, dp, p

    columnsInSegment = zeros(Int64,numVarTypes,1)
    columnsInSegment[1] = numLines
    columnsInSegment[2] = numLines
    columnsInSegment[3] = numLines
    columnsInSegment[4] = numNodes

    return(columnsInSegment)
end

function variableMapping(numLines, numNodes, columnsInSegment)
    # create the variable mapping as a 2 D array
    # map[row,column] = global index
    # row is the type of variable (m, m2, dp, p)
    # column contains the index for each of the available variables in the global vector
    # function to count number of equations and variables
    numEqnTypes = 5 # not used
    numVarTypes = 4 # m, m2, dp, p

    maxOtherIndex = numLines + numNodes
    globalColumn = zeros(Int64,numVarTypes,maxOtherIndex)

    index = 1
    for i = 1:size(columnsInSegment)[1]
        for j = 1:columnsInSegment[i]
            globalColumn[i,j] = index
            index += 1
        end
    end

    return(globalColumn)
end

function doNetworkHydraulics(lineHydraulics, fluidList, connectivity, dofList, nodeList, fittingList)
    # there is a lot of code here. I do not want the user typing this. It should only be function calls
    #
    # check input data
        # numLines = checkSegments(lineHydraulics)
        # numNodes = checkNodes
        # I have a DOF check in the code
        # or numLines = getSize(lineHydraulics)
        # or numNodes = getSize(nodeList)
        # variableMapping = getVariableMapping(numLines, numNodes)
    #
    # resultVector = doHydraulics(lineHydraulics, fluidList,dofList,nodeList)
    # Then we need to create a new table with the segment results
    # and we need another table with the pressures
    #
    # then do this
    # segmentResults = extractResults("segment", lineHydraulics, resultVector)
    # nodeResults = extractResults("node", lineHydraulics, resultVector)
    # I could supply a list of dataframe names to include in the results
    #

    # driver script
    addFluidProperties(lineHydraulics,fluidList)
    addPipeProperties(lineHydraulics)

    # and this is where we need to initialize
    # return the matrix of mass flow and m2
    # I will need to provide the connectivity df as well to get all of the fields
    massFlow = initializeMassFlow(lineHydraulics,connectivity) # if we do not have a mass flow, make one up

    # pass the mass flow matrix
    #and set the Reynolds number and friction factor to a reasonable value
    hydraulicParameters = initializeHydraulicParameters(40000.0, lineHydraulics)

    numLines = size(connectivity)[1]
    numResistance = size(lineHydraulics)[1]
    numNodes = size(nodeList)[1];
    #C = zeros(Float64,numLines,numNodes)

    segList = copy(connectivity[:,:Segment])
#    segList = copy(lineHydraulics[:,:Segment])
    nList = copy(nodeList[:,:Node])

    # connectivity, used for pressure continuity
    # each row defines a segment: we have the inlet and outlet node
    # I don't think I need the lineHydraulics df
    Amatrix = getConnectivity(connectivity,nodeList)
    # I think I only need to assemble the block matrix.
    Ia = zeros(Float64,numLines,numLines)
    for i = 1:numLines
        Ia[i,i] = 1.0;
    end;

    # create the mass balance matrix
    Bprelim = Amatrix'
    Bmatrix = removeTrivialRows(Bprelim);

    # I need to create the constraint equations
    # use the connectivity df instead of lineHydraulics
    Cm = getConstraintMatrix(dofList, "flowRate", connectivity, nodeList)
    Cp = getConstraintMatrix(dofList, "pressure", connectivity, nodeList)
    xc = getConstraintMatrix(dofList, "value", connectivity, nodeList);

    # need to define where each matrix will go.
    # sure I could move this outside the loop, but it should not slow things down much
    # need vectors for the number of rows and columns in each segment
    # I could put most of this batch of code in a function
    # let
    # l = num lines
    # n = num nodes
    # k = resistance calcs
    # d = degrees of freedom

    # num unknonws = 3*l + n
    # count equations
    # num of mass balance relations = num rows in B
    # quadratic equations = l
    # resistance eqns = k
    # pressure connectivity = l
    # degree of freedom = d

        # the block matrix is
        #  1.  2.  3.  4.
        #    m, m2, dp,  p | rhs
        # 1  B,  0,  0,  0 | 0
        # 2 Qm,Qm2,  0,  0 | xq
        # 3  0,-Km,Iseg, 0 | 0
        # 4  0,  0, -I, Am | 0
        # 5 Cm,  0,  0, Cp | xc

    # function to count number of equations and variables
    # edit this to account fo rnumber of resistances
    numEqnTypes = 5
    numVarTypes = 4
    rowsInSegment = zeros(Int64,numEqnTypes,1)
    rowsInSegment[1] = size(Bmatrix)[1]
    rowsInSegment[2] = numLines
    rowsInSegment[3] = numResistance
    rowsInSegment[4] = numLines
    rowsInSegment[5] = size(dofList)[1]

    columnsInSegment = zeros(Int64,numVarTypes,1)
    columnsInSegment[1] = numLines
    columnsInSegment[2] = numLines
    columnsInSegment[3] = numLines
    columnsInSegment[4] = numNodes

    numEquations = sum(rowsInSegment)
    numVariables = sum(columnsInSegment)

    # I need this screening as a preliminary check
    (numEquations != numVariables) ? error("error: ", numEquations, " equations and ", numVariables, " variables") : println("OK: ", numEquations, " equations and variables")

    # this dozen lines could go into a function
    # I could create a matrix with global indices
    # globalRow[segment][localRow]
    # globalColumn[segment][localColumn]
    maxOtherIndex = numLines + numNodes
    globalRow = zeros(Int64,numEqnTypes,maxOtherIndex)
    globalColumn = zeros(Int64,numVarTypes,maxOtherIndex)

    index = 1
    for i = 1:size(rowsInSegment)[1]
        for j = 1:rowsInSegment[i]
            globalRow[i,j] = index
            index += 1
        end
    end

    index = 1
    for i = 1:size(columnsInSegment)[1]
        for j = 1:columnsInSegment[i]
            globalColumn[i,j] = index
            index += 1
        end
    end

    # define the global block matrix and vector

    nGlobalRow = maximum(globalRow)
    nGlobalColumn = maximum(globalColumn)
    gBM = zeros(Float64,nGlobalRow,nGlobalColumn) # global block matrix
    gBv = zeros(Float64,nGlobalRow,1) # global block vector
    gBx = zeros(Float64,nGlobalRow,1) # global block solution

    epsM = 0.01 # mass flow kg/h
    maxIter = 15

    errM = 100.0
    iter = 0

    # and this is where we need to iterate

    while ((errM > epsM) && (iter < maxIter) )
        # the resistance matrix is not applied to segments that are control valves
        ourElementK = getElementK(hydraulicParameters, fittingList)
        Km = getResistanceMatrix(hydraulicParameters, ourElementK, connectivity)
        Iseg = getSegmentIMatrix(hydraulicParameters, connectivity)

        # we need the quadratic matrix.
        # must realize that our lineHydraulic table is not the definative source of infomration
        # for this
        # we should use the connectivity table
        # Or do we create a simple table for only mass flow and m2?
        # m2 = 2 m M - M M
        # m2 - 2 m M = -M M
        # change this from lineHydraulics to massFlow
        # why does massFlow not work?
#        Qm2 = getQuadraticMatrix(lineHydraulics, "quadratic")
#        Qm  = getQuadraticMatrix(lineHydraulics, "linear")
#        xq  = getQuadraticMatrix(lineHydraulics, "rhs");

        Qm2 = getQuadraticMatrix(massFlow, "quadratic")
        Qm  = getQuadraticMatrix(massFlow, "linear")
        xq  = getQuadraticMatrix(massFlow, "rhs");

        # the block matrix is
        #  1.  2.  3.  4.
        #    m, m2, dp,  p | rhs
        # 1  B,  0,  0,  0 | 0
        # 2 Qm,Qm2,  0,  0 | xq
        # 3  0,-Km,Iseg, 0 | 0
        # 4  0,  0, -I, Am | 0
        # 5 Cm,  0,  0, Cp | xc
        #
        # P1 - P2 = K.m2, where P1 is in, P2 is out
        # DP = K.m2
        # P1 - P2 = DP
        # or
        # -K.m2 + DP = 0
        # -DP + P1 - P2 = 0

        # once I have the answer, how do I get inlet and outlet pressure for each segment.
            # The A matrix contains the information
            # The rows in A are for each segment.
            # columns with +1 are inlet, -1 are outlet.
            # Transform the A matrix by extracting the entries with +1. Then matrix multiply.
            # then get the entries with -1 and matrix multiply.

        # assemble the global block matrix
        i = 1
        j = 1
        addToBlockMatrix(gBM,Bmatrix,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])

        i = 2
        j = 1
        addToBlockMatrix(gBM,Qm,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])

        i = 2
        j = 2
        addToBlockMatrix(gBM,Qm2,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])
        addToBlockRHS(gBv, xq, globalRow, i, rowsInSegment[i])

        i = 3
        j = 2
        addToBlockMatrix(gBM,-Km,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])

        i = 3
        j = 3
        addToBlockMatrix(gBM,Iseg,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])

        i = 4
        j = 3
        addToBlockMatrix(gBM,-Ia,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])

        i = 4
        j = 4
        addToBlockMatrix(gBM,Amatrix,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])

        i = 5
        j = 1
        addToBlockMatrix(gBM,Cm,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])

        i = 5
        j = 4
        addToBlockMatrix(gBM,Cp,globalRow,globalColumn,i,j,rowsInSegment[i],columnsInSegment[j])
        addToBlockRHS(gBv, xc, globalRow, i, rowsInSegment[i])

        # and solve
        testSoln = gBM \ gBv

        # convergnece
#        errM = maximum(abs.(lineHydraulics[:,:massFlow] - testSoln[1:numLines]))

        errM = maximum(abs.(massFlow[:,:massFlow] - testSoln[1:numLines]))

        iter += 1

        # update the mass flow rate solutions
        massFlow[:,:massFlow] = testSoln[1:numLines]
        for i = 1:numResistance
            ii = indexLookup(lineHydraulics[i,:Segment], connectivity[:,:Segment])
            lineHydraulics[i,:massFlow] = testSoln[ii]
        end

        # then I need to update the hydraulic parameters
        hydraulicParameters = Hydraulics2.getReynolds(lineHydraulics)
        # and I need to update the pressures and DP estimates

        gBx = testSoln
    end
    return (gBx)
end

# create routines to extract results
# extractFromBlockSolution(testSolution, globalColumn, j (which type is it),columnsInSegment[j])
function combineResults(dfAllResults, dfLines)
    # dfAll has all of the rows
    # dfLines has the pressur drop hydraulic info
    # do an innerjoin to produce a single table

    hydraulicDetails = (Hydraulics2.getReynolds(dfLines));
    combined1=outerjoin(dfAllResults, hydraulicDetails, on = intersect(names(dfAllResults), names(hydraulicDetails)), makeunique=false,indicator=nothing,validate=(false,false));
    combined2=outerjoin(combined1, dfLines, on = intersect(names(combined1), names(dfLines)), makeunique=false,indicator=nothing,validate=(false,false));

    return(combined2)
end

end # module


function ArcsSet(arg0::BSPTree, arg1::jdouble)
    return ArcsSet((BSPTree, jdouble), arg0, arg1)
end

function ArcsSet(arg0::Collection, arg1::jdouble)
    return ArcsSet((Collection, jdouble), arg0, arg1)
end

function ArcsSet(arg0::jdouble)
    return ArcsSet((jdouble,), arg0)
end

function ArcsSet(arg0::jdouble, arg1::jdouble, arg2::jdouble)
    return ArcsSet((jdouble, jdouble, jdouble), arg0, arg1, arg2)
end

function apply_transform(obj::AbstractRegion, arg0::Transform)
    return jcall(obj, "applyTransform", AbstractRegion, (Transform,), arg0)
end

function as_list(obj::ArcsSet)
    return jcall(obj, "asList", List, ())
end

function build_new(obj::ArcsSet, arg0::BSPTree)
    return jcall(obj, "buildNew", ArcsSet, (BSPTree,), arg0)
end

function check_point(obj::AbstractRegion, arg0::JVector)
    return jcall(obj, "checkPoint", Region_Location, (JVector,), arg0)
end

function check_point(obj::AbstractRegion, arg0::Point)
    return jcall(obj, "checkPoint", Region_Location, (Point,), arg0)
end

function contains(obj::AbstractRegion, arg0::Region)
    return jcall(obj, "contains", jboolean, (Region,), arg0)
end

function copy_self(obj::AbstractRegion)
    return jcall(obj, "copySelf", AbstractRegion, ())
end

function for_each(obj::Iterable, arg0::Consumer)
    return jcall(obj, "forEach", void, (Consumer,), arg0)
end

function get_barycenter(obj::AbstractRegion)
    return jcall(obj, "getBarycenter", Point, ())
end

function get_boundary_size(obj::AbstractRegion)
    return jcall(obj, "getBoundarySize", jdouble, ())
end

function get_size(obj::AbstractRegion)
    return jcall(obj, "getSize", jdouble, ())
end

function get_tolerance(obj::AbstractRegion)
    return jcall(obj, "getTolerance", jdouble, ())
end

function get_tree(obj::AbstractRegion, arg0::jboolean)
    return jcall(obj, "getTree", BSPTree, (jboolean,), arg0)
end

function intersection(obj::AbstractRegion, arg0::SubHyperplane)
    return jcall(obj, "intersection", SubHyperplane, (SubHyperplane,), arg0)
end

function is_empty(obj::AbstractRegion)
    return jcall(obj, "isEmpty", jboolean, ())
end

function is_empty(obj::AbstractRegion, arg0::BSPTree)
    return jcall(obj, "isEmpty", jboolean, (BSPTree,), arg0)
end

function is_full(obj::AbstractRegion)
    return jcall(obj, "isFull", jboolean, ())
end

function is_full(obj::AbstractRegion, arg0::BSPTree)
    return jcall(obj, "isFull", jboolean, (BSPTree,), arg0)
end

function iterator(obj::ArcsSet)
    return jcall(obj, "iterator", JIterator, ())
end

function project_to_boundary(obj::ArcsSet, arg0::Point)
    return jcall(obj, "projectToBoundary", BoundaryProjection, (Point,), arg0)
end

function split(obj::ArcsSet, arg0::Arc)
    return jcall(obj, "split", ArcsSet_Split, (Arc,), arg0)
end

function spliterator(obj::Iterable)
    return jcall(obj, "spliterator", Spliterator, ())
end


# Users of this module should first read the document
# https://github.com/alan-turing-institute/MLJ.jl/blob/master/doc/adding_new_models.md

module MLJBase

export MLJType, Model, Supervised, Unsupervised, Deterministic, Probabilistic
export Rows, Cols, Schema, retrieve, getrows
export fit, update, clean!, info, coerce
export predict, predict_mean, predict_mode
export transform, inverse_transform, se, evaluate, best
export target_kind, target_quantity, inputs_can_be, is_pure_julia
export package_url, package_name, package_uuid

export HANDLE_GIVEN_ID, @show, @constant  # from show.jl
export UnivariateNominal, average         # from distributions.jl

# methods from other packages to be rexported:
export pdf, mean, mode

import Base.==
using Query
import TableTraits
import DataFrames                # TODO: get rid of this dependency
import Distributions
import Distributions: pdf, mode
using CategoricalArrays

# from Standard Library:
using Statistics

## CONSTANTS

# the directory containing this file:
const srcdir = dirname(@__FILE__)
# horizontal space for field names in `MLJType` object display:
const COLUMN_WIDTH = 24
# how deep to display fields of `MLJType` objects:
const DEFAULT_SHOW_DEPTH = 1

## ABSTRACT TYPES

# overarching MLJ type:
abstract type MLJType end

# for storing hyperparameters:
abstract type Model <: MLJType end

abstract type Supervised{R} <: Model end # parameterized by fit-result type `R`
abstract type Unsupervised <: Model  end

# supervised models that `predict` probability distributions are of:
abstract type Probabilistic{R} <: Supervised{R} end

# supervsied models that `predict` point-values are of:
abstract type Deterministic{R} <: Supervised{R} end

# for displaying objects of `MLJType`:
include("show.jl")

# probability distributions and methods not provided by
# Distributions.jl package:
include("distributions.jl")

# convenience methods for manipulating categorical and tabular data
include("data.jl")

## THE MODEL INTERFACE

# every model interface must implement a `fit` method of the form
# `fit(model, verbosity, X, y) -> fitresult, cache, report` or
# `fit(model, verbosity, X, ys...) -> fitresult, cache, report` (multivariate case)
# or, one the simplified versions
# `fit(model, X, y) -> fitresult`
# `fit(model, X, ys...) -> fitresult`
fit(model::Model, verbosity::Int, args...) = fit(model, args...), nothing, nothing

# each model interface may optionally overload the following refitting
# method:
update(model::Model, verbosity, fitresult, cache, args...) =
    fit(model, verbosity, args...)

# methods dispatched on a model and fit-result are called *operations*.
# supervised models must implement this operation:
function predict end

# probabilistic supervised models may also overload one or more of
# these operations delevering point predictions:
predict_mode(model::Supervised, fitresult, Xnew) =
    mode.(predict(model, fitresult, Xnew))
predict_mean(model::Supervised, fitresult, Xnew) =
    mean.(predict(model, fitresult, Xnew))
predict_median(model::Supervised, fitresult, Xnew) =
    median.(predict(model, fitresult, Xnew))

# unsupervised methods must implement this operation:
function transform end

# unsupervised methods may implement this operation:
function inverse_transform end

# operations implemented by some meta-models:
function se end
function evaluate end
function best end

# a model wishing invalid hyperparameters to be corrected with a
# warning should overload this method (return value is the warning
# message):
clean!(model::Model) = ""

# supervised models may need to overload the following method to
# ensure the input data supplied by user (something implementing the
# Queryverse iterable table API) is coerced into the form required by
# its `fit` method and operations:
coerce(model::Model, Xtable) = Xtable

# if the return type `TABLE` of `coerce` is not a Queryverse
# iterable table or a `Matrix` (with rows
# corresponding to patterns and columns corresponding to features)
# then users will not be able to use MLJ's performant `EnsembleModel`
# on `model` unless one overloads the following method for type
# `TABLE`:
getrows(model::Model, X, r) = retrieve(X, Rows, r)
# here `r` is any integer, unitrange or colon `:`, and the right-hand
# side defined in `data.jl`.

# fallback trait declarations:
target_kind(::Type{<:Supervised}) = :unknown
target_quantity(::Type{<:Supervised}) = :univariate
inputs_can_be(::Type{<:Model}) = Symbol[]
is_pure_julia(::Type{<:Model}) = :unknown
package_name(::Type{<:Model}) = "unknown"
package_uuid(::Type{<:Model}) = "unknown"
package_url(::Type{<:Model}) = "unknown"

_response(::Type{<:Supervised}) = :unknown
_response(::Type{<:Deterministic}) = :deterministic
_response(::Type{<:Probabilistic}) = :probabilistic

target_is(modeltype::Type{<:Supervised}) =
    [_response(modeltype), target_kind(modeltype), target_quantity(modeltype)]

if VERSION < v"1.0.0"
    import Base.info
end

function info(modeltype::Type{<:Model})

    message = "$modeltype has a bad trait declaration."

    if modeltype <: Supervised
        target_kind(modeltype) in [:numeric, :binary, :multiclass, :unknown] ||
            error(message*"target_kind must return :numeric, :binary, :multiclass (or :unknown).")
        target_quantity(modeltype) in [:univariate, :multivariate] ||
            error(message*"target_quantity must return :univariate or :multivariate")
    end

    issubset(Set(inputs_can_be(modeltype)), Set([:numeric, :nominal, :missing])) ||
        error(message*"inputs_can_be must return a vector with entries from [:numeric, :nominal, :missing]")
    is_pure_julia(modeltype) in [:yes, :no, :unknown] ||
        error(message*"is_pure_julia must return :yes, :no (or :unknown).")

    # modelnamesplit = split(string(modeltype.name), '.')
    # if length(modelnamesplit) > 1
    #     modelnamesplit = modelnamesplit[2:end]
    # end
    # modelname = string(reduce((a, b)->"$a.$b", modelnamesplit))

    d = Dict{Symbol,Union{Symbol,Vector{Symbol},String}}()
    d[:model_name] = string(modeltype)
    d[:inputs_can_be] = inputs_can_be(modeltype)
    d[:is_pure_julia] = is_pure_julia(modeltype)
    d[:package_name] = package_name(modeltype)
    d[:package_uuid] = package_uuid(modeltype)
    d[:package_url] = package_url(modeltype)
    if modeltype <: Supervised
        d[:target_is] = target_is(modeltype)
    end

    return d
end
info(model::Supervised) = info(typeof(model))

# models are `==` if they have the same type and their field values are `==`:
function ==(m1::M, m2::M) where M<:Model
    ret = true
    for fld in fieldnames(M)
        ret = ret && getfield(m1, fld) == getfield(m2, fld)
    end
    return ret
end

end # module


using Revise, MDToolbox, Plots, JLD2, Printf, DelimitedFiles

function create_image(pdbFileNum, quatenionNum, saveFileName)
    # read structure and remove center of mass
    pdb = MDToolbox.readpdb("model/0000$(pdbFileNum).pdb")
    MDToolbox.decenter!(pdb)

    # read quaternions
    quater = DelimitedFiles.readdlm("quaternion/QUATERNION_LIST_576_Orient")

    # rotate by to a quaternion value
    pdb_rotated = MDToolbox.rotate(pdb, quater[quatenionNum, :])
    afmData = MDToolbox.afmize(pdb_rotated, (150.0, 150.0), (16, 16))
    @save saveFileName afmData
end

pdbFiles = [1 2 3 4]
quatenionNums = [7, 52, 91, 500]
fileNameList = []
for i in 1:4
    saveFileName = @sprintf "testCase/pdb%02d_quaternion%04d.jld2" pdbFiles[i] quatenionNums[i]
    create_image(pdbFiles[i], quatenionNums[i], saveFileName)
    push!(fileNameList, saveFileName)
end
@save "testCase/fileNameList.jld2" fileNameList


CRCBS.build_env(solver,pc_mapf::AbstractPC_MAPF,env::SearchEnv,node::ConstraintTreeNode,i::Int) = build_env(solver,pc_mapf,env,node,AgentID(i))
function construct_heuristic_model(solver::PIBTPlanner,env_graph;
        ph = PerfectHeuristic(get_dist_matrix(env_graph)),
        kwargs...)
    construct_composite_heuristic(ph,ph,NullHeuristic())
end
function construct_cost_model(solver::PIBTPlanner,
        schedule, cache, problem_spec, env_graph, primary_objective=SumOfMakeSpans(); kwargs...)
    cost_model = construct_composite_cost_model(
        typeof(primary_objective)(schedule,cache),
        FullCostModel(maximum,TravelTime()),
        FullCostModel(maximum,TravelDistance())
        )
    heuristic_model = construct_heuristic_model(solver,env_graph;kwargs...)
    cost_model, heuristic_model
end

function CRCBS.is_consistent(cache::PIBTCache,pc_mapf::PC_MAPF)
    search_env = CRCBS.get_solution(cache)
    n_jobless = 0
    # for e in CRCBS.get_envs(cache)
    #     if !CRCBS.is_valid(get_goal(e))
    #         n_jobless += 1
    #     end
    # end
    if length(get_cache(search_env).closed_set) + n_jobless >= nv(get_schedule(search_env))
        return true
    elseif !any(CRCBS.is_valid, map(e->CRCBS.get_goal(e),CRCBS.get_envs(cache)))
        # all robots are done with their jobs
        return true
    end
    return false
end

function CRCBS.pibt_priority_law(solver,pc_mapf::PC_MAPF,cache,i)
    env = CRCBS.get_envs(cache)[i]
    return (
        ~CRCBS.is_valid(env,get_goal(env)),
        ~isa(get_node(env),DEPOSIT),
        ~isa(get_node(env),CARRY),
        ~isa(get_node(env),COLLECT),
        # minimum(cache.solution.cache.slack[get_vtx(
        #     cache.solution.schedule,env.node_id)]),
        minimum(get_slack(cache.solution,env.node_id)),
        -CRCBS.get_timers(cache)[i],
        i
    )
end

function CRCBS.pibt_preprocess!(solver,pc_mapf::PC_MAPF,cache)
    update_planning_cache!(solver,CRCBS.get_solution(cache))
    CRCBS.pibt_update_envs!(solver,pc_mapf,cache)
end

"""
    CRCBS.pibt_update_solution!(solver,pc_mapf::PC_MAPF,solution::SearchEnv,cache)

Overridden to update the SearchEnv
"""
function CRCBS.pibt_update_solution!(solver,solution::SearchEnv,cache)
    # update route plan
    CRCBS.pibt_update_solution!(solver,get_route_plan(solution),cache)
    # NOTE Update planning cache so that all goal times can be updated
    update_planning_cache!(solver,solution)
    solution
end
function pibt_info_string(cache,i,
        env=CRCBS.get_envs(cache)[i],
        s=CRCBS.get_states(cache)[i],
        a=CRCBS.get_actions(cache)[i]
    )
    string("\t",i,": ", get_vtx(cache.solution.schedule,env.node_id)," ",
        string(get_node(env))," s=",string(s),", a=",string(a),
        ", goal=",string(env.goal),"\n")
end
function pibt_info_strings(cache)
    info_strings = String[]
    for i in CRCBS.get_ordering(cache)
        str = pibt_info_string(cache,i)
        push!(info_strings,str)
    end
    return info_strings
end
function CRCBS.pibt_update_envs!(solver,pc_mapf::PC_MAPF,cache)
    @log_info(2,verbosity(solver),"PIBT iteration $(iterations(solver)) PRE UPDATE:\n",
        pibt_info_strings(cache)...)
    solution = CRCBS.get_solution(cache)
    node = initialize_root_node(solver,pc_mapf)
    done = true
    sweeps = 0
    while true
        done = true
        for (i,p) in enumerate(get_paths(solution))
            v_next = get_next_vtx_matching_agent_id(solution,RobotID(i))
            if has_vertex(solution.schedule,v_next)
                # rebuild env to ensure that the goal time is up to date
                CRCBS.get_envs(cache)[i] = build_env(solver,pc_mapf,solution,node,AgentID(i))
            end
            env = CRCBS.get_envs(cache)[i]
            v = get_vtx(solution.schedule,env.node_id)
            sp = get_final_state(p)
            if (v in solution.cache.active_set) && is_goal(env,sp) && CRCBS.is_valid(env,get_goal(env))
                done = false
                update_env!(solver,solution,v,p)
                update_planning_cache!(solver,solution)
                v_next = get_next_vtx_matching_agent_id(solution,RobotID(i))
                @assert v_next != v
                if has_vertex(solution.schedule,v_next)
                    CRCBS.get_timers(cache)[i] = 0
                end
            end
        end
        sweeps += 1
        done ? break : nothing
    end
    @log_info(2,verbosity(solver),"PIBT iteration $(iterations(solver)) POST UPDATE:\n",
        pibt_info_strings(cache)...)
    return cache
end


import JLBoost: feature_importance

import MLJBase: CrossEntropy

CrossEntropy(x, y::CategoricalVector) = CrossEntropy(x, y.refs .- 1)


type TopographicPoint2D <: Singleton
  Heatmap::Array{Float64,2}
  radiusOfInterest::Float64
end

function evalPotential(topo::TopographicPoint2D, Xi::Array{Graphs.ExVertex,1}; N::Int=100)
  warn("evalPotential TopographicPoint2D is not finished yet")
  return topo.Heatmap
end


#permutation routine function-
#general logic: receive array of permutation parameters, until a model more likely than the least is found:
#randomly select a model from the ensemble (the least likely having been removed by this point), then sample new models by permuting with each of hte given parameter sets until a model more likely than the current contour is found
#if none is found for the candidate model, move on to another candidate until the models_to_permute iterate is reached, after which return nothing for an error code

#four permutation modes: source (iterative random changes to sources until model lh>contour or iterate limit reached)
#						-(iterates, weight shift freq per source base, length change freq per source, weight_shift_dist (a ContinuousUnivariateDistribution)) for permute params
#						mix (iterative random changes to mix matrix as above)
#						-(iterates, unitrange of # of moves)
#						init (iteratively reinitialize sources from priors)
#						-(iterates) for init params
#						merge (iteratively copy a source + mix matrix row from another model in the ensemble until lh>contour or iterate						limit reached)
#						-(iterates) for merpge params

function run_permutation_routine(e::Bayes_IPM_ensemble, job_sets::Vector{Tuple{Vector{Tuple{String,Any}},Vector{AbstractFloat}}}, job_set_thresh::Vector{AbstractFloat}, job_limit::Integer, models_to_permute::Integer, contour::AbstractFloat)
    start=time()
    job_set,job_weights=job_sets[findlast(thresh->(contour>thresh),job_set_thresh)]
    !(length(job_weights)==length(job_set)) && throw(ArgumentError("Job set and job weight vec must be same length!"))

	for model = 1:models_to_permute
		m_record = rand(e.models)
        m = deserialize(m_record.path)

        for job in 1:job_limit
            mode, params = get_job(job_set,job_weights,m.flags,m.sources,m.source_length_limits)
            if mode == "PS"
                new_m=permute_source(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
            elseif mode == "PM"
                new_m=permute_mix(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
            elseif mode == "PSFM"
				new_m=perm_src_fit_mix(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
            elseif mode == "FM"
                new_m=fit_mix(m, e.obs_array, e.obs_lengths, e.bg_scores)
            elseif mode == "DM"
                new_m=distance_merge(e.models, m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
            elseif mode == "SM"
                new_m=similarity_merge(e.models, m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
			elseif mode == "RD"
				new_m=random_decorrelate(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
			elseif mode == "RI"
				new_m=reinit_src(m, contour,  e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, e.mix_prior, params...)
            elseif mode == "EM"
                new_m=erode_model(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                occursin("PSFM",new_m.flags[1]) && (mode="PSFM")
            else
                @error "Malformed permute mode code! Current supported: \"PS\", \"PM\", \"PSFM\", \"FM\", \"DM\", \"SM\",\"RD\", \"RI\", \"EM\""
            end

			dupecheck(new_m,m) && new_m.log_Li > contour && return new_m, (time()-start, job, model, m.log_Li, new_m.log_Li, mode)
		end
	end
	return nothing, nothing
end

function worker_permute(e::Bayes_IPM_ensemble, job_chan::RemoteChannel, models_chan::RemoteChannel, job_sets::Vector{Tuple{Vector{Tuple{String,Any}},Vector{AbstractFloat}}}, job_set_thresh::Vector{AbstractFloat}, job_limit::Integer, models_to_permute::Integer)
	persist=true
    id=myid()
    model_ctr=1
	while persist
        wait(job_chan)

        start=time()

		models = fetch(job_chan)
        models === nothing && (persist=false) && break
		contour, ll_idx = findmin([model.log_Li for model in models])
		deleteat!(models, ll_idx)
        job_set,job_weights=job_sets[findlast(thresh->(contour>thresh),job_set_thresh)]
        !(length(job_weights)==length(job_set)) && throw(ArgumentError,"Job set and job weight vec must be same length!")

		for model=1:models_to_permute
			found::Bool=false
			m_record = rand(models)
			m = remotecall_fetch(deserialize,1,m_record.path)
            for job in 1:job_limit
                mode, params = get_job(job_set,job_weights,m.flags,m.sources,m.source_length_limits)
                if mode == "PS"
                    new_m=permute_source(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                elseif mode == "PM"
                    new_m=permute_mix(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                elseif mode == "PSFM"
                    new_m=perm_src_fit_mix(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                elseif mode == "FM"
                    new_m=fit_mix(m, e.obs_array, e.obs_lengths, e.bg_scores)
                elseif mode == "DM"
                    new_m=distance_merge(e.models, m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                elseif mode == "SM"
                    new_m=similarity_merge(e.models, m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                elseif mode == "RD"
                    new_m=random_decorrelate(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                elseif mode == "RI"
                    new_m=reinit_src(m, contour,  e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, e.mix_prior, params...)
                elseif mode == "EM"
                    new_m=erode_model(m, contour, e.obs_array, e.obs_lengths, e.bg_scores, e.source_priors, params...)
                    occursin("PSFM",new_m.flags[1]) && (mode="PSFM")
                else
                    @error "Malformed permute mode code! Current supported: \"PS\", \"PM\", \"PSFM\", \"FM\", \"DM\", \"SM\",\"RD\", \"RI\", \"EM\""
                end
				dupecheck(new_m,m) && new_m.log_Li > contour && (put!(models_chan, (new_m ,id, (time()-start, job, model_ctr, m.log_Li, new_m.log_Li, mode))); found=true; model_ctr=1; break)
			end
            found==true && break;
            model_ctr+=1
            wait(job_chan)
            fetch(job_chan)!=models && (break) #if the ensemble has changed during the search, update it
			model==models_to_permute && (put!(models_chan,nothing);persist=false)#worker to put nothing on channel if it fails to find a model more likely than contour
		end
	end
end

                function get_job(job_set, job_weights, flags, srcs, src_lims)
                    job_dist=Categorical(job_weights)
                    mode,params=job_set[rand(job_dist)]

                    if ("nofit" in flags)
                        while mode=="FM"
                            mode, params =job_set[rand(job_dist)]
                        end
                    end

                    if mode=="EM" && !any([>(size(srcs[s][1],1), src_lims[1]) for s in 1:length(srcs)])
                        while mode=="EM"
                            mode,params=job_set[rand(job_dist)]
                        end
                    end
                    return mode,params
                end

                function dupecheck(new_model, model)
                    (new_model.sources==model.sources && new_model.mix_matrix==model.mix_matrix) ? (return false) : (return true)
                end


module Flux

using Base: tail
using LinearAlgebra, Statistics, Random  # standard lib
using MacroTools, Reexport, ProgressLogging, SpecialFunctions
using MacroTools: @forward

@reexport using NNlib
using MLUtils
import Optimisers: Optimisers, trainable, destructure  # before v0.13, Flux owned these functions

using Zygote, ChainRulesCore
using Zygote: Params, @adjoint, gradient, pullback, @nograd
export gradient

# Pirate error to catch a common mistake. (Internal function `base` because overloading `update!` is more likely to give ambiguities.)
Optimisers.base(dx::Zygote.Grads) = error("Optimisers.jl cannot be used with Zygote.jl's implicit gradients, `Params` & `Grads`")

export Chain, Dense, Maxout, SkipConnection, Parallel,
       RNN, LSTM, GRU, GRUv3,
       SamePad, Conv, CrossCor, ConvTranspose, DepthwiseConv,
       AdaptiveMaxPool, AdaptiveMeanPool, GlobalMaxPool, GlobalMeanPool, MaxPool, MeanPool,
       Dropout, AlphaDropout, LayerNorm, BatchNorm, InstanceNorm, GroupNorm,
       Upsample, PixelShuffle,
       fmap, cpu, gpu, f32, f64,
       testmode!, trainmode!

include("optimise/Optimise.jl")
using .Optimise
using .Optimise: @epochs
using .Optimise: skip
export Descent, ADAM, Momentum, Nesterov, RMSProp,
  ADAGrad, AdaMax, ADADelta, AMSGrad, NADAM, OADAM,
  ADAMW, RADAM, AdaBelief, InvDecay, ExpDecay,
  WeightDecay, ClipValue, ClipNorm

using CUDA
const use_cuda = Ref{Union{Nothing,Bool}}(nothing)

include("utils.jl")
include("onehot.jl")
include("functor.jl")

# Pirate error to catch a common mistake.
Functors.functor(::Type{<:MLUtils.DataLoader}, x) = error("`DataLoader` does not support Functors.jl, thus functions like `Flux.gpu` will not act on its contents.")

include("layers/stateless.jl")
include("layers/basic.jl")
include("layers/conv.jl")
include("layers/recurrent.jl")
include("layers/normalise.jl")
include("layers/upsample.jl")
include("layers/show.jl")

include("loading.jl")

include("outputsize.jl")

include("data/Data.jl")
using .Data

include("losses/Losses.jl")
using .Losses # TODO: stop importing Losses in Flux's namespace in v0.12

include("deprecations.jl")

include("cuda/cuda.jl")

end # module


## ExoplanetsSysSim/src/kepler_catalog.jl
## (c) 2015 Eric B. Ford

#using ExoplanetsSysSim
using DataFrames
#using DataArrays
using CSV
#using JLD
#using JLD2
using FileIO

#if VERSION >= v"0.5-"
#  import Compat: UTF8String, ASCIIString
#end

mutable struct KeplerPhysicalCatalog
  target::Array{KeplerTarget,1}
end
#KeplerPhysicalCatalog() = KeplerPhysicalCatalog([])

mutable struct KeplerObsCatalog
  target::Array{KeplerTargetObs,1}
end
KeplerObsCatalog() = KeplerObsCatalog(KeplerTargetObs[])

"""
    generate_kepler_physical_catalog(sim_param)

Wrapper function to create catalog of simulated Kepler targets.

# Arguments:
- `sim_param::SimParam`: Simulation parameter object; must have the following parameters set:
  - num_targets_sim_pass_one = Number of Kepler targets in simulated catalog
  - generate_kepler_target = Function which generates Kepler targets
  - (Optional) stellar_catalog = Stellar catalog filename
  - (Optional) star_table_setup = Function that loads stellar catalog into DataFrame

# Returns:
Kepler physical catalog object containing all simulated Kepler targets and associated planetary systems.
"""
function generate_kepler_physical_catalog(sim_param::SimParam)
   if haskey(sim_param,"stellar_catalog")
      star_tab_func = get_function(sim_param, "star_table_setup")
      star_tab_func(sim_param)
   end
   num_sys = get_int(sim_param,"num_targets_sim_pass_one")
   generate_kepler_target = get_function(sim_param,"generate_kepler_target")
   target_list = Array{KeplerTarget}(undef,num_sys)
   map!(x->generate_kepler_target(sim_param), target_list, 1:num_sys )
   return KeplerPhysicalCatalog(target_list)
end

"""
    observe_kepler_targets_sky_avg(input, sim_param)

Wrapper function to create catalog of simulated observations of Kepler targets using the sky averaging observation scheme (i.e. each planet's detection probability is the average detection probability over all view-angles).

# Arguments:
- `input::KeplerPhysicalCatalog`: Catalog object of simulated Kepler targets and associated planetary systems to be observed
- `sim_param::SimParam`: Simulation parameter object; requires the following simulation parameters to be set:
  - calc_target_obs_sky_ave: Function name for sky averaging simulated observations

# Returns:
Kepler observations catalog object containing all properties observed from the Kepler targets and associated planetary systems that were detected during the simulation.
"""
function observe_kepler_targets_sky_avg(input::KeplerPhysicalCatalog, sim_param::SimParam )
  calc_target_obs = get_function(sim_param,"calc_target_obs_sky_ave")
  return observe_kepler_targets(calc_target_obs, input, sim_param)
end

"""
    observe_kepler_targets_single_obs(input, sim_param)

Wrapper function to create catalog of simulated observations of Kepler targets using the single observer observation scheme (i.e. each planet's detection probability is the detection probability from the Earth).

# Arguments:
- `input::KeplerPhysicalCatalog`: Catalog object of simulated Kepler targets and associated planetary systems to be observed
- `sim_param::SimParam`: Simulation parameter object; requires the following simulation parameters to be set:
  - calc_target_obs_single_obs: Function name for single observer simulated observations

# Returns:
Kepler observations catalog object containing all properties observed from the Kepler targets and associated planetary systems that were detected during the simulation.
"""
function observe_kepler_targets_single_obs(input::KeplerPhysicalCatalog, sim_param::SimParam )
  calc_target_obs = get_function(sim_param,"calc_target_obs_single_obs")
  return observe_kepler_targets(calc_target_obs, input, sim_param)
end

"""
    observe_kepler_targets(calc_target_obs, input, sim_param)

Wrapper function to create catalog of simulated observations of Kepler targets.

# Arguments:
- `calc_target_obs::Function`: Function to use in simulating observations of Kepler targets (sky averaging vs. single observer schemes)
- `input::KeplerPhysicalCatalog`: Catalog object of simulated Kepler targets and associated planetary systems to be observed
- `sim_param::SimParam`: Simulation parameter object

# Returns:
Kepler observations catalog object containing all properties observed from the Kepler targets and associated planetary systems that were detected during the simulation.
"""
function observe_kepler_targets(calc_target_obs::Function, input::KeplerPhysicalCatalog, sim_param::SimParam )
  #calc_target_obs = get_function(sim_param,"calc_target_obs_sky_ave")
  #calc_target_obs = get_function(sim_param,"calc_target_obs_single_obs")
  output = KeplerObsCatalog()
  if haskey(sim_param,"mem_kepler_target_obs")
     output.target = get(sim_param,"mem_kepler_target_obs",Array{KeplerTargetObs}(0) )
  end
  num_targets_sim_pass_one = get_int(sim_param,"num_targets_sim_pass_one")
  if length(output.target) < num_targets_sim_pass_one
     output.target = Array{KeplerTargetObs}(undef,num_targets_sim_pass_one)
  end
  #output.target = Array{KeplerTargetObs}(undef,length(input.target) )  # Replaced to reduce memory allocation
  map!(x::KeplerTarget->calc_target_obs(x,sim_param)::KeplerTargetObs, output.target, input.target)
  resize!(output.target,length(input.target))
  return output
end

# Test if this planetary system has at least one planet that transits (assuming a single observer)
function select_targets_one_obs(ps::PlanetarySystemAbstract)
## REFACTOR MARKER: NOT KEPLER SPECIFIC
 for pl in 1:length(ps.orbit)
   ecc::Float64 = ps.orbit[pl].ecc
   incl::Float64 = ps.orbit[pl].incl
   a::Float64 = semimajor_axis(ps,pl)
   Rstar::Float64 = rsol_in_au*ps.star.radius
   if (Rstar > (a*(1-ecc)*(1+ecc))/(1+ecc*sin(ps.orbit[pl].omega))*cos(incl))
     return true
   end
 end
 return false
end
#=
function select_targets_one_obs(ps::PlanetarySystemAbstract)
  for pl in 1:length(ps.orbit)
    if does_planet_transit(ps,pl)
       return true
    end
  end
  return false
end
=#

# Remove undetected planets from physical catalog
# TODO: OPT: Maybe create array of bools for which planets to keep, rather than splicing out non-detections?
function generate_obs_targets(cat_phys::KeplerPhysicalCatalog, sim_param::SimParam )
  for t in 1:length(cat_phys.target)
    kep_targ = cat_phys.target[t]
    for ps in 1:length(cat_phys.target[t].sys)
      sys = kep_targ.sys[ps]
      for pl in length(sys.orbit):-1:1    # Going in reverse since removing planets from end of list first is cheaper than starting at beginning
        ecc::Float64 = sys.orbit[pl].ecc
	incl::Float64 = sys.orbit[pl].incl
   	a::Float64 = semimajor_axis(sys,pl)
   	Rstar::Float64 = rsol_in_au*sys.star.radius

        does_it_transit = does_planet_transit(sys, pl)
        pdet_if_tr = does_it_transit ? calc_prob_detect_if_transit_with_actual_b(kep_targ, ps, pl, sim_param) : 0.
        if !does_it_transit || (rand()>pdet_if_tr)
    	  splice!(cat_phys.target[t].sys[ps].orbit, pl)
	  splice!(cat_phys.target[t].sys[ps].planet, pl)
     	end
      end
    end
  end
  return cat_phys
end

# The following function is primarily left for debugging.
# Create a catalog of observations of simulated Kepler targets.
function simulated_read_kepler_observations(sim_param::SimParam )
   println("# WARNING: Using simulated_read_kepler_observations.")
   # if haskey(sim_param,"stellar_catalog")
   #    star_tab_func = get_function(sim_param, "star_table_setup")
   #    star_tab_func(sim_param)
   # end
   num_sys = get_int(sim_param,"num_kepler_targets")
   generate_kepler_target = get_function(sim_param,"generate_kepler_target")
   target_list = Array{KeplerTarget}(undef,num_sys)
   map!(x->generate_kepler_target(sim_param), target_list, 1:num_sys )

   cat_phys_cut = generate_obs_targets(KeplerPhysicalCatalog(target_list), sim_param)
   calc_target_obs = get_function(sim_param,"calc_target_obs_single_obs")
   output = KeplerObsCatalog()
   output.target = map(x::KeplerTarget->calc_target_obs(x,sim_param)::KeplerTargetObs, cat_phys_cut.target)
   return output
end

"""
    read_koi_catalog(sim_param, force_reread=false)

Wrapper function to read Kepler Object of Interest (KOI) catalog given SimParam

# Arguments:
- `sim_param::SimParam`: Simulation parameter object; this function uses the following parameters from the SimParam object:
  - planetary_catalog: String filename of Kepler Object of Interest catalog (if not provided, defaults to "q1_q17_dr25_koi.csv"
- `force_reread::Bool`: Should the file be read in even if a DataFrame of the KOIs already exists in workspace?

# Returns:
- Dataframe of KOI objects (and their respective properties).
- Vector of booleans indicating which KOIs  were designated as planet candidates by the Kepler pipeline and have a valid observed radius ratio and period (necessary for detection probability calculation).
"""
function read_koi_catalog(sim_param::SimParam, force_reread::Bool = false)
    filename = convert(String,joinpath(dirname(pathof(ExoplanetsSysSim)),"..", "data", convert(String,get(sim_param,"planetary_catalog","q1_q17_dr25_koi.csv")) ) )
    return read_koi_catalog(filename, force_reread)
end

"""
    read_koi_catalog(filename, force_reread=false)

Function to read Kepler Object of Interest (KOI) catalog given filename string.

# Arguments:
- `filename::String`: String filename of Kepler Object of Interest catalog
- `force_reread::Bool`: Should the file be read in even if a DataFrame of the KOIs already exists in workspace?

# Returns:
- Dataframe of KOI objects (and their respective properties).
- Vector of booleans indicating which KOIs  were designated as planet candidates by the Kepler pipeline and have a valid observed radius ratio and period (necessary for detection probability calculation).
"""
function read_koi_catalog(filename::String, force_reread::Bool = false)
    local df, usable

    if occursin(r".jld2$",filename) && !force_reread
        try
            data = load(filename)
            df = data["planetary_catalog"]
            usable = data["koi_catalog_usable"]
            Core.typeassert(df,DataFrame)
            Core.typeassert(usable,Array{Int64,1})
        catch
            error(string("# Failed to read koi catalog >",filename,"< in jld2 format."))
        end
    else
       try
            df = CSV.read(filename,comment="#")

            # Choose which KOIs to keep
            #is_cand = (csv_data[!,:,koi_disposition_idx] .== "CONFIRMED") | (csv_data[!,:,koi_disposition_idx] .== "CANDIDATE")
            is_cand = df[!,:koi_pdisposition] .== "CANDIDATE"
            has_radius = .!ismissing.(df[!,:koi_ror])
            has_period = .!(ismissing.(df[!,:koi_period]) .| ismissing.(df[!,:koi_period_err1]) .| ismissing.(df[!,:koi_period_err2]))

            is_usable = .&(is_cand, has_radius, has_period)
            usable = findall(is_usable)
           #  symbols_to_keep = [:kepid, :kepoi_name, :koi_pdisposition, :koi_score, :koi_ror, :koi_period, :koi_period_err1, :koi_period_err2, :koi_time0bk, :koi_time0bk_err1, :koi_time0bk_err2, :koi_depth, :koi_depth_err1, :koi_depth_err2, :koi_duration, :koi_duration_err1, :koi_duration_err2]
           # df = df[usable, symbols_to_keep]
           # tmp_df = DataFrame()
           # for col in names(df)
           #     tmp_df[col] = collect(skipmissing(df[col]))
           # end
           # df = tmp_df
           # usable = collect(1:length(df[!,:kepid]))
        catch
            error(string("# Failed to read koi catalog >",filename,"< in ascii format."))
        end
    end
    return df, usable
end

"""
  setup_actual_pc_catalog_kepler(df_star, df_koi, usable_koi, sim_param)

Create (true) catalog of Kepler observations of Kepler targets

# Arguments:
- `df_star::DataFrame`: DataFrame containing all Kepler target stars in catalog
   NOTE: df_star is assumed to have fields kepid, mass and radius for all targets in the survey)
- `df_koi::DataFrame`: DataFrame containing all Kepler Object of Interests (KOIs)
- `usable_koi::Array{Integer}`: Array of KOI dataframe row indices corresponding to KOIs to use
- `sim_param::SimParam`: Simulation parameter object

# Returns:
- Kepler observations catalog containing Kepler targets and associated KOIs (to be used as true catalog in comparison with simulated observations)
"""
function setup_actual_pc_catalog_kepler(df_star::DataFrame, df_koi::DataFrame, usable_koi::Array{Int64}, sim_param::SimParam)
    local target_obs, num_pl
    df_koi = df_koi[usable_koi,:]

    # Deprecated code to take a list of KepIDs and KOI names to pre-select a subset of KOIs
    # if haskey(sim_param, "koi_subset_csv")
    #     koi_subset = fill(false, length(df_koi[!,:kepid]))

    #     subset_df = readtable(convert(String,get(sim_param,"koi_subset_csv", "christiansen_kov.csv")), header=true, separator=' ')

    #     for n in 1:length(subset_df[!,1])
    #         subset_colnum = 1
    #         subset_entry = findall(x->x==subset_df[n,1], df_koi[names(subset_df)[1]])
    #         # println("Initial cut: ", subset_entry)
    #         while (length(subset_entry) > 1) & (subset_colnum < length(names(subset_df)))
    #             subset_colnum += 1

    #             subsubset = findall(x->round(x*10.)==round(subset_df[n,subset_colnum]*10.), df_koi[subset_entry,names(subset_df)[subset_colnum]])
    #             # println("Extra cut: ", subset_df[n,subset_colnum], " / ", df_koi[subset_entry,col_idx], " = ", subsubset)
    #             subset_entry = subset_entry[subsubset]
    #         end
    #         if length(subset_entry) > 1
    #             cand_sub = findall(x->x == "CANDIDATE",df_koi[subset_entry,:koi_pdisposition])
    #             subset_entry = subset_entry[cand_sub]
    #             if length(subset_entry) > 1
    #                 println("# Multiple planets found in final cut: ", subset_df[n,1])
    #             end
    #         end
    #         if length(subset_entry) < 1
    #             println("# No planets found in final cut: ", subset_df[n,:])
    #         end
    #         koi_subset[subset_entry] = true
    #     end
    #     df_koi = df_koi[findall(koi_subset),:]
    #     tot_plan = count(x->x, koi_subset)
    # end

    output = KeplerObsCatalog()
    sort!(df_star, (:kepid))
    df_obs = innerjoin(df_star, df_koi, on = :kepid)
    #df_obs = sort!(df_obs, cols=(:kepid))
    df_obs = sort!(df_obs, (:kepid))

    # if haskey(sim_param, "koi_subset_csv")
    #     tot_plan -= length(df_obs[!,:kepoi_name])
    #     println("# Number of planet candidates in subset file with no matching star in table: ", tot_plan)
    # end

    # Add each KOI planet candidate to Kepler target object
    plid = 0
    for i in 1:length(df_obs[!,:kepoi_name])
        if plid == 0
            plid = 1
            while i+plid < length(df_obs[!,:kepoi_name]) && df_obs[i+plid,:kepid] == df_obs[i,:kepid]
                plid += 1
            end
            num_pl = plid
            target_obs = KeplerTargetObs(num_pl)
	        #target_obs.star = ExoplanetsSysSim.StarObs(df_obs[i,:radius],df_obs[i,:mass],findfirst(df_star[!,:kepid], df_obs[i,:kepid]))
            star_idx = searchsortedfirst(df_star[!,:kepid],df_obs[i,:kepid])
            if star_idx > length(df_star[!,:kepid])
                @warn "# Couldn't find kepid " * df_star[i,:kepid] * " in df_obs."
                star_idx = rand(1:length(df_star[!,:kepid]))
            end
            target_obs.star = ExoplanetsSysSim.StarObs(df_obs[i,:radius],df_obs[i,:mass],star_idx)

        end

        target_obs.obs[plid] = ExoplanetsSysSim.TransitPlanetObs(df_obs[i,:koi_period],df_obs[i,:koi_time0bk],df_obs[i,:koi_depth]/1.0e6,df_obs[i,:koi_duration])
        target_obs.sigma[plid] = ExoplanetsSysSim.TransitPlanetObs((abs(df_obs[i,:koi_period_err1])+abs(df_obs[i,:koi_period_err2]))/2,(abs(df_obs[i,:koi_time0bk_err1])+abs(df_obs[i,:koi_time0bk_err2]))/2,(abs(df_obs[i,:koi_depth_err1]/1.0e6)+abs(df_obs[i,:koi_depth_err2]/1.0e6))/2,(abs(df_obs[i,:koi_duration_err1])+abs(df_obs[i,:koi_duration_err2]))/2)
	#target_obs.prob_detect = ExoplanetsSysSim.SimulatedSystemDetectionProbs{OneObserver}( ones(num_pl), ones(num_pl,num_pl), ones(num_pl), fill(Array{Int64}(undef,0), 1) )  # Made line below to simplify calling
        target_obs.prob_detect = ExoplanetsSysSim.OneObserverSystemDetectionProbs(num_pl)
        plid -= 1
        if plid == 0
            push!(output.target,target_obs)
        end
    end
    return output
end

# Two functions below were just for debugging purposes
# Calculate SNR of every planet in simulated catalog
function calc_snr_list(cat::KeplerPhysicalCatalog, sim_param::SimParam)
  snrlist = Array{Float64}(undef,0)
  for t in 1:length(cat.target)
    for p in 1:length(cat.target[t].sys[1].planet)
      snr = calc_snr_if_transit(cat.target[t],1,p,sim_param)
      if snr>0.0
        push!(snrlist,snr)
      end
    end
  end
  snrlist[findall(x->x>7.1,snrlist)]
end

# Calculate detection probability (assuming planet transits) for every planet in simulated catalog
function calc_prob_detect_list(cat::KeplerPhysicalCatalog, sim_param::SimParam)
  pdetectlist = Array{Float64}(undef,0)
  for t in 1:length(cat.target)
    for p in 1:length(cat.target[t].sys[1].planet)
      #pdet = calc_prob_detect_if_transit(cat.target[t],1,p,sim_param)
      pdet = calc_prob_detect_if_transit_with_actual_b(cat.target[t],1,p,sim_param)

      if pdet>0.0
        push!(pdetectlist,pdet)
      end
    end
  end
  idx = findall(x->x>0.0,pdetectlist)
  pdetectlist[idx]
end

function test_catalog_constructors(sim_param::SimParam)
  ## REFACTOR MARKER: NOT KEPLER SPECIFIC
  # or maybe it is if it _seriously_ depends on CDPPs
  cat_phys = generate_kepler_physical_catalog(sim_param)::KeplerPhysicalCatalog
  id = findfirst( x->num_planets(x)>=1 , cat_phys.target)   # fast forward to first target that has some planets
  @assert(length(id)>=1)
  semimajor_axis(cat_phys.target[id].sys[1],1)
  pdetlist = calc_prob_detect_list(cat_phys,sim_param)
  calc_target_obs_single_obs(cat_phys.target[id],sim_param)
  calc_target_obs_sky_ave(cat_phys.target[id],sim_param)
  @assert( length(cat_phys.target[id].sys[1].planet)  == num_planets(cat_phys.target[id]) )
  cat_obs = simulated_read_kepler_observations(sim_param)
  return (cat_phys, cat_obs)
end


@everywhere module PCBM  # defined on all workers
    type Model
        rxns
        mets
        genes
        S
        lb
        ub
        c
        b
        csense
        rxn_gene_mat
        rxn_name
        rxn_rules
        rxn_subsystem
        rxn_extra
        met_formula
        met_name
        met_extra
        gene_name
        gene_extra
        description
    end

    solvers = map(x -> in(x, readdir(Pkg.dir())) ? x : "" , ["GLPK", "CPLEX"])

    function set_solver(solver)
        Solver = ""
        if solver == "glpk"
            Solver = "GLPK"
        elseif solver == "cplex"
            Solver = "CPLEX"
        else
            warn("choose one of the following solvers:")
            warn("glpk")
            warn("cplex")
            return
        end

        global settings_default_lp = solver
    end

    if in("GLPK", solvers)
        using GLPK
        include(Pkg.dir() * "/CBM/src/core/solvers/GLPK.jl")
        global settings_default_lp = "glpk"
    end

    if in("CPLEX", solvers)
        using CPLEX
        include(Pkg.dir() * "/CBM/src/core/solvers/CPLEX.jl")
    end

    if !isdefined(:settings_default_lp)
        if in("GLPK", solvers)
            global settings_default_lp = "glpk"
        elseif in("CPLEX", solvers)
            global settings_default_lp = "cplex"
        else
            warn("Neither GLPK or CPLEX is installed")
        end
    end

    function global_setup_lp{T <: Number}(c::Array{T}, lb::Array{T}, ub::Array{T}, b_lower::Array{T}, b_upper::Array{T}, S::SparseMatrixCSC, objective::AbstractString = "max")
        global lp = PCBM.setup_lp(c,lb,ub,b_lower, b_upper, S, objective)
    end

    function setup_lp{T <: Number}(c::Array{T}, lb::Array{T}, ub::Array{T}, b_lower::Array{T}, b_upper::Array{T}, S::SparseMatrixCSC, objective::AbstractString = "max")
        if settings_default_lp == "glpk"
            return PCBM.setup_glpk(c,lb,ub,b_lower, b_upper, S, objective)
        elseif settings_default_lp == "cplex"
            return PCBM.setup_cplex(c,lb,ub,b_lower, b_upper, S, objective)
        else
            warn("No solver")
        end
    end

    ###############################################################################
    # stuff for FVA in Parallel
    ###############################################################################
    # THINK setup_workers is perhaps uneccesary check when you have time
    function setup_workers(model)
        @sync for pid in workers()
            @async remotecall_fetch(PCBM.global_setup_lp, pid,
                zeros(Float64, length(model.rxns)),
                    model.lb,
                    model.ub,
                    model.b,
                    model.b,
                    model.S)
        end
    end

    function fast_fva_objective(x)
        res = Array(Float64, length(x))

        for (i,v) in enumerate(x)
            PCBM.set_objective(lp, v, 1.0)

            PCBM.solve(lp)
            res[i] = PCBM.get_objective_value(lp)

            PCBM.set_objective(lp, v, 0.0)
        end

        return res
    end

    function fast_fva_solution(x)
        res = Array(Float64, length(get_variable_bounds(lp)[1]), length(x))

        for (i,v) in enumerate(x)
            PCBM.set_objective(lp, v, 1.0)

            PCBM.solve(lp)
            res[:,i] = PCBM.get_solution(lp)

            PCBM.set_objective(lp, v, 0.0)
        end

        return res
    end

    function fast_fva(model, optPercentage::Number = 1, flux_matrix::Bool = false)
        nrxns = length(model.rxns)
        x = collect(1:nrxns)

        lpmin = setup_lp(model.c, model.lb, model.ub, model.b,model.b, model.S)
        solve(lpmin)
        wt_min = get_objective_value(lpmin)

        lb = deepcopy(model.lb)
        lb[find(model.c)] *= wt_min

        @sync for pid in workers()
            @async remotecall_fetch(PCBM.global_setup_lp, pid,
                zeros(Float64, length(model.rxns)),
                    lb,
                    model.ub,
                    model.b,
                    model.b,
                    model.S)
        end

        stepsize = div(nrxns, 3*length(workers()))
        checklist = map(x -> collect(x:x+stepsize-1), 1:stepsize:nrxns)
        checklist[end] = checklist[end][checklist[end] .<= nrxns]

        maxF = flux_matrix ?  pmap(PCBM.fast_fva_solution, checklist) : pmap(PCBM.fast_fva_objective, checklist)

        map(x -> remotecall_fetch(() -> PCBM.set_direction(lp, "min"), x), workers())

        minF = flux_matrix ?  pmap(PCBM.fast_fva_solution, checklist) : pmap(PCBM.fast_fva_objective, checklist)

        if flux_matrix
            maxV = hcat(maxF...)
            minV = hcat(minF...)
            maxF = diag(maxV)
            minF = diag(minV)
            return minF, maxF, minV, maxV
        end

        return vcat(minF...), vcat(maxF...)
    end
    ###############################################################################
    ###############################################################################

    ###############################################################################
    # stuff for gene deletion in Parallel
    ###############################################################################
    function setup_workers_for_gene_deletion(model)
        @sync for pid in workers()
            @async remotecall_fetch(PCBM.global_setup_lp, pid,
                model.c,
                model.lb,
                model.ub,
                model.b,
                model.b,
                model.S)
        end
    end

    function pgd_analysis(rxns)
        #progress_meter = ProgressMeter.Progress(length(rxns), 0.5, "\n\u1b[1FFVA:", 50)
        combos = sort(rxns, lt=lexless)
        num_combos = length(combos)

        solve(lp)
        wt = get_objective_value(lp)

        lb, ub = get_variable_bounds(lp)

        flow = Array(Float64, num_combos)

        ####
        for d in combos[1]
            set_col_bounds(lp, d, 0.0, 0.0)
        end

        solve(lp)
        flow[1] = get_objective_value(lp)

        disable = []
        for i in 2:num_combos
            enable = setdiff(combos[i-1], combos[i])
            disable = setdiff(combos[i], combos[i-1])

            for e in enable
                set_col_bounds(lp, e, lb[e], ub[e])
            end

            for d in disable
                set_col_bounds(lp, d, 0.0, 0.0)
            end
            solve(lp)
            flow[i] = get_objective_value(lp)
            #next!(progress_meter)
        end

        for e in disable
            set_col_bounds(lp, e, lb[e], ub[e])
        end

        flow /= wt
        flow = round(flow, 16)

        result = Dict(zip(combos, flow))

        map(x -> set_col_bounds(lp, x, lb[x], ub[x]), 1:length(lb))

        return result
    end

    function pgd(model, rxns)
        listlen = length(rxns)
        setup_workers_for_gene_deletion(model)

        stepsize = div(listlen, 3*length(workers()))
        checklist = map(x -> collect(x:x+stepsize-1), 1:stepsize:listlen)
        checklist[end] = checklist[end][checklist[end] .<= listlen]

        rxnlist = map(x -> rxns[x], checklist)

        res = pmap(PCBM.pgd_analysis, rxnlist)
    end
    ###############################################################################
    ###############################################################################

end