# ----------------------------------------------------------------------------------- #
# Copyright (c) 2016 Varnerlab
# Robert Frederick Smith School of Chemical and Biomolecular Engineering
# Cornell University, Ithaca NY 14850
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
# ----------------------------------------------------------------------------------- #
#
# ----------------------------------------------------------------------------------- #
# Function: DataDictionary
# Description: Holds simulation and model parameters as key => value pairs in a Julia Dict()
# Generated on: 2016-11-02T07:00:00.307
#
# Input arguments:
# time_start::Float64 => Simulation start time value (scalar)
# time_stop::Float64 => Simulation stop time value (scalar)
# time_step::Float64 => Simulation time step (scalar)
#
# Output arguments:
# data_dictionary::Dict{AbstractString,Any} => Dictionary holding model and simulation parameters as key => value pairs
# ----------------------------------------------------------------------------------- #
function DataDictionary(time_start::Float64,time_stop::Float64,time_step_size::Float64)

	# stoichiometric_matrix and dilution_matrix -
	stoichiometric_matrix = readdlm("./Network.dat")
	dilution_matrix = readdlm("./Dilution.dat")
	degradation_matrix = readdlm("./Degradation.dat")

	# array of gene lengths -
	# gene_coding_length_array = [
	# 	15000	;	# 1	gene_AP1
	# 	15000	;	# 2	gene_AhR
	# 	15000	;	# 3	gene_CD11b
	# 	15000	;	# 4	gene_CD14
	# 	15000	;	# 5	gene_CD38
	# 	15000	;	# 6	gene_CEBPa
	# 	15000	;	# 7	gene_E2F
	# 	15000	;	# 8	gene_EGR1
	# 	15000	;	# 9	gene_GFI1
	# 	15000	;	# 10	gene_IRF1
	# 	15000	;	# 11	gene_OCT1
	# 	15000	;	# 12	gene_OCT4
	# 	15000	;	# 13	gene_P21
	# 	15000	;	# 14	gene_P47Phox
	# 	15000	;	# 15	gene_PPARg
	# 	15000	;	# 16	gene_PU1
	# 	15000	;	# 17	gene_Trigger
	# 	15000	;	# 18	gene_cRAF
	# ]

	gene_coding_length_array = [
		10323	;	# 1	gene_AP1
		47530	;	# 2	gene_AhR
		72925	;	# 3	gene_CD11b
		8974	;	# 4	gene_CD14
		74978	;	# 5	gene_CD38
		2630	;	# 6	gene_CEBPa
		17919	;	# 7	gene_E2F
		10824	;	# 8	gene_EGR1
		13833	;	# 9	gene_GFI1
		16165	;	# 10	gene_IRF1
		206516 	;	# 11	gene_OCT1
		6356	;	# 12	gene_OCT4
		15651	;	# 13	gene_P21
		3074	;	# 14	gene_P47Phox https://www.ncbi.nlm.nih.gov/nuccore/AF003533.1   #1475	;	# 14	gene_P47Phox -- https://www.ncbi.nlm.nih.gov/nuccore/NM_000265.5
		153507	;	# 15	gene_PPARg
		40782	;	# 16	gene_PU1
		5908	;	# 17	gene_Trigger
		87571; # 18	gene_cRAF 	https://www.ncbi.nlm.nih.gov/nuccore/NG_007467.1 #3291	;	# 18	gene_cRAF --- NCBI homo sapien: https://www.ncbi.nlm.nih.gov/nuccore/NM_002880.3
	];

	# array of mRNA coding lengths -
	mRNA_coding_length_array = [
		gene_coding_length_array[1]	;	# 19	1	mRNA_gene_AP1
		gene_coding_length_array[2]	;	# 20	2	mRNA_gene_AhR
		gene_coding_length_array[3]	;	# 21	3	mRNA_gene_CD11b
		gene_coding_length_array[4]	;	# 22	4	mRNA_gene_CD14
		gene_coding_length_array[5]	;	# 23	5	mRNA_gene_CD38
		gene_coding_length_array[6]	;	# 24	6	mRNA_gene_CEBPa
		gene_coding_length_array[7]	;	# 25	7	mRNA_gene_E2F
		gene_coding_length_array[8]	;	# 26	8	mRNA_gene_EGR1
		gene_coding_length_array[9]	;	# 27	9	mRNA_gene_GFI1
		gene_coding_length_array[10]	;	# 28	10	mRNA_gene_IRF1
		gene_coding_length_array[11]	;	# 29	11	mRNA_gene_OCT1
		gene_coding_length_array[12]	;	# 30	12	mRNA_gene_OCT4
		gene_coding_length_array[13]	;	# 31	13	mRNA_gene_P21
		gene_coding_length_array[14]	;	# 32	14	mRNA_gene_P47Phox
		gene_coding_length_array[15]	;	# 33	15	mRNA_gene_PPARg
		gene_coding_length_array[16]	;	# 34	16	mRNA_gene_PU1
		gene_coding_length_array[17]	;	# 35	17	mRNA_gene_Trigger
		gene_coding_length_array[18]	;	# 36	18	mRNA_gene_cRAF
	]

	# array of mRNA coding lengths -
	# protein_coding_length_array = [
	# 	round((0.33)*mRNA_coding_length_array[1])	;	# 37	1	protein_gene_AP1
	# 	round((0.33)*mRNA_coding_length_array[2])	;	# 38	2	protein_gene_AhR
	# 	round((0.33)*mRNA_coding_length_array[3])	;	# 39	3	protein_gene_CD11b
	# 	round((0.33)*mRNA_coding_length_array[4])	;	# 40	4	protein_gene_CD14
	# 	round((0.33)*mRNA_coding_length_array[5])	;	# 41	5	protein_gene_CD38
	# 	round((0.33)*mRNA_coding_length_array[6])	;	# 42	6	protein_gene_CEBPa
	# 	round((0.33)*mRNA_coding_length_array[7])	;	# 43	7	protein_gene_E2F
	# 	round((0.33)*mRNA_coding_length_array[8])	;	# 44	8	protein_gene_EGR1
	# 	round((0.33)*mRNA_coding_length_array[9])	;	# 45	9	protein_gene_GFI1
	# 	round((0.33)*mRNA_coding_length_array[10])	;	# 46	10	protein_gene_IRF1
	# 	round((0.33)*mRNA_coding_length_array[11])	;	# 47	11	protein_gene_OCT1
	# 	round((0.33)*mRNA_coding_length_array[12])	;	# 48	12	protein_gene_OCT4
	# 	round((0.33)*mRNA_coding_length_array[13])	;	# 49	13	protein_gene_P21
	# 	round((0.33)*mRNA_coding_length_array[14])	;	# 50	14	protein_gene_P47Phox
	# 	round((0.33)*mRNA_coding_length_array[15])	;	# 51	15	protein_gene_PPARg
	# 	round((0.33)*mRNA_coding_length_array[16])	;	# 52	16	protein_gene_PU1
	# 	round((0.33)*mRNA_coding_length_array[17])	;	# 53	17	protein_gene_Trigger
	# 	round((0.33)*mRNA_coding_length_array[18])	;	# 54	18	protein_gene_cRAF
	# ]

	# array of mRNA coding lengths -
	protein_coding_length_array = [
		331	;	# 37	1	protein_gene_AP1
		848	;	# 38	2	protein_gene_AhR
		1153	;	# 39	3	protein_gene_CD11b
		375	;	# 40	4	protein_gene_CD14
		300	;	# 41	5	protein_gene_CD38
		393	;	# 42	6	protein_gene_CEBPa
		437	;	# 43	7	protein_gene_E2F
		543	;	# 44	8	protein_gene_EGR1
		422	;	# 45	9	protein_gene_GFI1
		325	;	# 46	10	protein_gene_IRF1
		741.3333333333	;	# 47	11	protein_gene_OCT1
		206.3333333	;	# 48	12	protein_gene_OCT4
		198	;	# 49	13	protein_gene_P21
		390	;	# 50	14	protein_gene_P47Phox -- https://www.ncbi.nlm.nih.gov/protein/NP_000256.4
		250	;	# 51	15	protein_gene_PPARg
		270.5	;	# 52	16	protein_gene_PU1
		420.6666667		;	# 53	17	protein_gene_Trigger
		648	;	# 54	18	protein_gene_cRAF --https://www.ncbi.nlm.nih.gov/protein/NP_002871.1
	]

	# ------------------------------------------------------------------------------------------#
	# constants (from bionumbers)       units
	# ------------------------------------------------------------------------------------------#
	cell_diameter = 12                  # mum
	number_of_rnapII = 75000            # copies/cells
	number_of_ribosome = 1e6            # copies/cells
	mRNA_half_life_TF = 2               # hrs
	protein_half_life = 10              # hrs
	doubling_time_cell = 19.5           # hrs
	max_translation_rate = 5            # aa/sec
	max_transcription_rate = 6.0        # nt/sec
	average_transcript_length = 15000   # nt
	average_protein_length = 5000       # aa
	fraction_nucleus = 0.49             # dimensionless
	av_number = 6.02e23                 # number/mol
	avg_gene_number = 2                 # number of copies of a gene
	# ------------------------------------------------------------------------------------------#
	#
	# ------------------------------------------------------------------------------------------#
	# Calculate constants using bionumber values
	# ------------------------------------------------------------------------------------------#
	# Calculate the volume (convert to L)
	V = ((1-fraction_nucleus)*(1/6)*(3.14159)*(cell_diameter)^3)*(1e-15)

	# Calculate the rnapII_concentration and ribosome_concentration
	rnapII_concentration = number_of_rnapII*(1/av_number)*(1/V)*1e9                   # nM
	ribosome_concentration = number_of_ribosome*(1/av_number)*(1/V)*1e9               # nM

	# degrdation rate constants -
	degradation_constant_mRNA = -(1/mRNA_half_life_TF)*log(0.5)                       # hr^-1
	degradation_constant_protein = -(1/protein_half_life)*log(0.5)                    # hr^-1

	# kcats for transcription and translation -
	kcat_transcription = max_transcription_rate*(3600/average_transcript_length)      # hr^-1
	kcat_translation = max_translation_rate*(3600/average_protein_length)             # hr^-1

	# Maximum specific growth rate -
	maximum_specific_growth_rate = (1/doubling_time_cell)*log(2)                      # hr^-1

	# What is the average gene concentration -
	avg_gene_concentration = avg_gene_number*(1/av_number)*(1/V)*1e9                  # nM

	# How fast do my cells die?
	death_rate_constant = 0.2*maximum_specific_growth_rate                            # hr^-1

	# Saturation constants for translation and trascription -
	saturation_transcription = 4600*(1/av_number)*(1/V)*1e9                           # nM
	saturation_translation = 100000*(1/av_number)*(1/V)*1e9                           # nM
	# -------------------------------------------------------------------------------------------#

	# initial condition array -
	initial_condition_array = [
		avg_gene_concentration	;	# 1	gene_AP1
		avg_gene_concentration	;	# 2	gene_AhR
		avg_gene_concentration	;	# 3	gene_CD11b
		avg_gene_concentration	;	# 4	gene_CD14
		avg_gene_concentration	;	# 5	gene_CD38
		avg_gene_concentration	;	# 6	gene_CEBPa
		avg_gene_concentration	;	# 7	gene_E2F
		avg_gene_concentration	;	# 8	gene_EGR1
		avg_gene_concentration	;	# 9	gene_GFI1
		avg_gene_concentration	;	# 10	gene_IRF1
		avg_gene_concentration	;	# 11	gene_OCT1
		avg_gene_concentration	;	# 12	gene_OCT4
		avg_gene_concentration	;	# 13	gene_P21
		avg_gene_concentration	;	# 14	gene_P47Phox
		avg_gene_concentration	;	# 15	gene_PPARg
		avg_gene_concentration	;	# 16	gene_PU1
		avg_gene_concentration	;	# 17	gene_Trigger
		avg_gene_concentration	;	# 18	gene_cRAF
		0.0	;	# 19	mRNA_gene_AP1
		0.0	;	# 20	mRNA_gene_AhR
		0.0	;	# 21	mRNA_gene_CD11b
		0.0	;	# 22	mRNA_gene_CD14
		0.0	;	# 23	mRNA_gene_CD38
		0.0	;	# 24	mRNA_gene_CEBPa
		0.0	;	# 25	mRNA_gene_E2F
		0.0	;	# 26	mRNA_gene_EGR1
		0.0	;	# 27	mRNA_gene_GFI1
		0.0	;	# 28	mRNA_gene_IRF1
		0.0	;	# 29	mRNA_gene_OCT1
		0.0	;	# 30	mRNA_gene_OCT4
		0.0	;	# 31	mRNA_gene_P21
		0.0	;	# 32	mRNA_gene_P47Phox
		0.0	;	# 33	mRNA_gene_PPARg
		0.0	;	# 34	mRNA_gene_PU1
		0.0	;	# 35	mRNA_gene_Trigger
		0.0	;	# 36	mRNA_gene_cRAF
		0.0	;	# 37	protein_gene_AP1
		0.0	;	# 38	protein_gene_AhR
		0.0	;	# 39	protein_gene_CD11b
		0.0	;	# 40	protein_gene_CD14
		0.0	;	# 41	protein_gene_CD38
		0.0	;	# 42	protein_gene_CEBPa
		0.0	;	# 43	protein_gene_E2F
		0.0	;	# 44	protein_gene_EGR1
		0.0	;	# 45	protein_gene_GFI1
		0.0	;	# 46	protein_gene_IRF1
		0.0	;	# 47	protein_gene_OCT1
		0.0	;	# 48	protein_gene_OCT4
		0.0	;	# 49	protein_gene_P21
		0.0	;	# 50	protein_gene_P47Phox
		0.0	;	# 51	protein_gene_PPARg
		0.0	;	# 52	protein_gene_PU1
		0.0	;	# 53	protein_gene_Trigger
		0.0	;	# 54	protein_gene_cRAF
	]

	binding_parameter_dictionary = Dict{AbstractString,Float64}()
	binding_parameter_dictionary["n_gene_AP1_gene_AhR"] = 1.0
	binding_parameter_dictionary["K_gene_AP1_gene_AhR"] = 120.0
	binding_parameter_dictionary["n_gene_AP1_gene_PU1"] = 1.0
	binding_parameter_dictionary["K_gene_AP1_gene_PU1"] = 120.0
	binding_parameter_dictionary["n_gene_AP1_gene_PPARg"] = 1.0
	binding_parameter_dictionary["K_gene_AP1_gene_PPARg"] = 120.0
	binding_parameter_dictionary["n_gene_AhR_gene_Trigger"] = 1.0
	binding_parameter_dictionary["K_gene_AhR_gene_Trigger"] = 120.0
	binding_parameter_dictionary["n_gene_CD11b_gene_PU1_gene_cRAF"] = 1.0
	binding_parameter_dictionary["K_gene_CD11b_gene_PU1_gene_cRAF"] = 120.0
	binding_parameter_dictionary["n_gene_CD14_gene_PPARg_gene_CEBPa_gene_EGR1_gene_cRAF"] = 1.0
	binding_parameter_dictionary["K_gene_CD14_gene_PPARg_gene_CEBPa_gene_EGR1_gene_cRAF"] = 120.0
	binding_parameter_dictionary["n_gene_CD38_gene_IRF1_gene_PPARg_gene_Trigger_gene_cRAF"] = 1.0
	binding_parameter_dictionary["K_gene_CD38_gene_IRF1_gene_PPARg_gene_Trigger_gene_cRAF"] = 120.0
	binding_parameter_dictionary["n_gene_CEBPa_gene_Trigger"] = 1.0
	binding_parameter_dictionary["K_gene_CEBPa_gene_Trigger"] = 120.0
	binding_parameter_dictionary["n_gene_CEBPa_gene_PPARg"] = 1.0
	binding_parameter_dictionary["K_gene_CEBPa_gene_PPARg"] = 120.0
	binding_parameter_dictionary["n_gene_CEBPa_gene_CEBPa"] = 1.0
	binding_parameter_dictionary["K_gene_CEBPa_gene_CEBPa"] = 120.0
	binding_parameter_dictionary["n_gene_CEBPa_gene_GFI1"] = 1.0
	binding_parameter_dictionary["K_gene_CEBPa_gene_GFI1"] = 120.0
	binding_parameter_dictionary["n_gene_E2F_gene_E2F"] = 1.0
	binding_parameter_dictionary["K_gene_E2F_gene_E2F"] = 120.0
	binding_parameter_dictionary["n_gene_E2F_gene_PPARg"] = 1.0
	binding_parameter_dictionary["K_gene_E2F_gene_PPARg"] = 120.0
	binding_parameter_dictionary["n_gene_E2F_gene_CEBPa"] = 1.0
	binding_parameter_dictionary["K_gene_E2F_gene_CEBPa"] = 120.0
	binding_parameter_dictionary["n_gene_E2F_gene_GFI1"] = 1.0
	binding_parameter_dictionary["K_gene_E2F_gene_GFI1"] = 120.0
	binding_parameter_dictionary["n_gene_E2F_gene_cRAF"] = 1.0
	binding_parameter_dictionary["K_gene_E2F_gene_cRAF"] = 120.0
	binding_parameter_dictionary["n_gene_EGR1_gene_Trigger"] = 1.0
	binding_parameter_dictionary["K_gene_EGR1_gene_Trigger"] = 120.0
	binding_parameter_dictionary["n_gene_EGR1_gene_PU1"] = 1.0
	binding_parameter_dictionary["K_gene_EGR1_gene_PU1"] = 120.0
	binding_parameter_dictionary["n_gene_EGR1_gene_PPARg"] = 1.0
	binding_parameter_dictionary["K_gene_EGR1_gene_PPARg"] = 120.0
	binding_parameter_dictionary["n_gene_EGR1_gene_GFI1"] = 1.0
	binding_parameter_dictionary["K_gene_EGR1_gene_GFI1"] = 120.0
	binding_parameter_dictionary["n_gene_GFI1_gene_CEBPa"] = 1.0
	binding_parameter_dictionary["K_gene_GFI1_gene_CEBPa"] = 120.0
	binding_parameter_dictionary["n_gene_GFI1_gene_EGR1"] = 1.0
	binding_parameter_dictionary["K_gene_GFI1_gene_EGR1"] = 120.0
	binding_parameter_dictionary["n_gene_IRF1_gene_Trigger"] = 1.0
	binding_parameter_dictionary["K_gene_IRF1_gene_Trigger"] = 120.0
	binding_parameter_dictionary["n_gene_IRF1_gene_AhR"] = 1.0
	binding_parameter_dictionary["K_gene_IRF1_gene_AhR"] = 120.0
	binding_parameter_dictionary["n_gene_IRF1_gene_PPARg"] = 1.0
	binding_parameter_dictionary["K_gene_IRF1_gene_PPARg"] = 120.0
	binding_parameter_dictionary["n_gene_OCT1_gene_PPARg"] = 1.0
	binding_parameter_dictionary["K_gene_OCT1_gene_PPARg"] = 120.0
	binding_parameter_dictionary["n_gene_OCT4_gene_Trigger"] = 1.0
	binding_parameter_dictionary["K_gene_OCT4_gene_Trigger"] = 120.0
	binding_parameter_dictionary["n_gene_OCT4_gene_AhR"] = 1.0
	binding_parameter_dictionary["K_gene_OCT4_gene_AhR"] = 120.0
	binding_parameter_dictionary["n_gene_OCT4_gene_cRAF"] = 1.0
	binding_parameter_dictionary["K_gene_OCT4_gene_cRAF"] = 120.0
	binding_parameter_dictionary["n_gene_P21_gene_Trigger_gene_AP1_gene_PPARg_gene_PU1_gene_IRF1_gene_CEBPa_gene_cRAF"] = 1.0
	binding_parameter_dictionary["K_gene_P21_gene_Trigger_gene_AP1_gene_PPARg_gene_PU1_gene_IRF1_gene_CEBPa_gene_cRAF"] = 120.0
	binding_parameter_dictionary["n_gene_P21_gene_GFI1"] = 1.0
	binding_parameter_dictionary["K_gene_P21_gene_GFI1"] = 120.0
	binding_parameter_dictionary["n_gene_P47Phox_gene_PU1_gene_CEBPa_gene_cRAF"] = 1.0
	binding_parameter_dictionary["K_gene_P47Phox_gene_PU1_gene_CEBPa_gene_cRAF"] = 120.0
	binding_parameter_dictionary["n_gene_P47Phox_gene_PPARg"] = 1.0
	binding_parameter_dictionary["K_gene_P47Phox_gene_PPARg"] = 120.0
	binding_parameter_dictionary["n_gene_PPARg_gene_Trigger"] = 1.0
	binding_parameter_dictionary["K_gene_PPARg_gene_Trigger"] = 120.0
	binding_parameter_dictionary["n_gene_PPARg_gene_CEBPa"] = 1.0
	binding_parameter_dictionary["K_gene_PPARg_gene_CEBPa"] = 120.0
	binding_parameter_dictionary["n_gene_PPARg_gene_EGR1"] = 1.0
	binding_parameter_dictionary["K_gene_PPARg_gene_EGR1"] = 120.0
	binding_parameter_dictionary["n_gene_PPARg_gene_PU1"] = 1.0
	binding_parameter_dictionary["K_gene_PPARg_gene_PU1"] = 120.0
	binding_parameter_dictionary["n_gene_PPARg_gene_AP1"] = 1.0
	binding_parameter_dictionary["K_gene_PPARg_gene_AP1"] = 120.0
	binding_parameter_dictionary["n_gene_PU1_gene_Trigger"] = 1.0
	binding_parameter_dictionary["K_gene_PU1_gene_Trigger"] = 120.0
	binding_parameter_dictionary["n_gene_PU1_gene_CEBPa"] = 1.0
	binding_parameter_dictionary["K_gene_PU1_gene_CEBPa"] = 120.0
	binding_parameter_dictionary["n_gene_PU1_gene_PU1"] = 1.0
	binding_parameter_dictionary["K_gene_PU1_gene_PU1"] = 120.0
	binding_parameter_dictionary["n_gene_PU1_gene_AP1"] = 1.0
	binding_parameter_dictionary["K_gene_PU1_gene_AP1"] = 120.0
	binding_parameter_dictionary["n_gene_PU1_gene_OCT1"] = 1.0
	binding_parameter_dictionary["K_gene_PU1_gene_OCT1"] = 120.0
	binding_parameter_dictionary["n_gene_PU1_gene_AhR"] = 1.0
	binding_parameter_dictionary["K_gene_PU1_gene_AhR"] = 120.0
	binding_parameter_dictionary["n_gene_PU1_gene_GFI1"] = 1.0
	binding_parameter_dictionary["K_gene_PU1_gene_GFI1"] = 120.0

	# Alias the control function parameters -
	control_parameter_dictionary = Dict{AbstractString,Float64}()
	control_parameter_dictionary["W_gene_AP1_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_AP1_gene_AhR"] = 1.0
	control_parameter_dictionary["W_gene_AP1_gene_PU1"] = 1.0
	control_parameter_dictionary["W_gene_AP1_gene_PPARg"] = 1.0
	control_parameter_dictionary["W_gene_AhR_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_AhR_gene_Trigger"] = 1.0
	control_parameter_dictionary["W_gene_CD11b_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_CD11b_gene_PU1_gene_cRAF"] = 1.0
	control_parameter_dictionary["W_gene_CD14_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_CD14_gene_PPARg_gene_CEBPa_gene_EGR1_gene_cRAF"] = 0.0
	control_parameter_dictionary["W_gene_CD38_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_CD38_gene_IRF1_gene_PPARg_gene_Trigger_gene_cRAF"] = 1.0
	control_parameter_dictionary["W_gene_CEBPa_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_CEBPa_gene_Trigger"] = 1.0
	control_parameter_dictionary["W_gene_CEBPa_gene_PPARg"] = 1.0
	control_parameter_dictionary["W_gene_CEBPa_gene_CEBPa"] = 1.0
	control_parameter_dictionary["W_gene_CEBPa_gene_GFI1"] = 1.0
	control_parameter_dictionary["W_gene_E2F_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_E2F_gene_E2F"] = 1.0
	control_parameter_dictionary["W_gene_E2F_gene_PPARg"] = 1.0
	control_parameter_dictionary["W_gene_E2F_gene_CEBPa"] = 1.0
	control_parameter_dictionary["W_gene_E2F_gene_GFI1"] = 1.0
	control_parameter_dictionary["W_gene_E2F_gene_cRAF"] = 1.0
	control_parameter_dictionary["W_gene_EGR1_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_EGR1_gene_Trigger"] = 1.0
	control_parameter_dictionary["W_gene_EGR1_gene_PU1"] = 1.0
	control_parameter_dictionary["W_gene_EGR1_gene_PPARg"] = 1.0
	control_parameter_dictionary["W_gene_EGR1_gene_GFI1"] = 1.0
	control_parameter_dictionary["W_gene_GFI1_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_GFI1_gene_CEBPa"] = 1.0
	control_parameter_dictionary["W_gene_GFI1_gene_EGR1"] = 1.0
	control_parameter_dictionary["W_gene_IRF1_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_IRF1_gene_Trigger"] = 1.0
	control_parameter_dictionary["W_gene_IRF1_gene_AhR"] = 1.0
	control_parameter_dictionary["W_gene_IRF1_gene_PPARg"] = 1.0
	control_parameter_dictionary["W_gene_OCT1_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_OCT1_gene_PPARg"] = 1.0
	control_parameter_dictionary["W_gene_OCT4_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_OCT4_gene_Trigger"] = 1.0
	control_parameter_dictionary["W_gene_OCT4_gene_AhR"] = 1.0
	control_parameter_dictionary["W_gene_OCT4_gene_cRAF"] = 1.0
	control_parameter_dictionary["W_gene_P21_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_P21_gene_Trigger_gene_AP1_gene_PPARg_gene_PU1_gene_IRF1_gene_CEBPa_gene_cRAF"] = 1.0
	control_parameter_dictionary["W_gene_P21_gene_GFI1"] = 1.0
	control_parameter_dictionary["W_gene_P47Phox_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_P47Phox_gene_PU1_gene_CEBPa_gene_cRAF"] = 1.0
	control_parameter_dictionary["W_gene_P47Phox_gene_PPARg"] = 1.0
	control_parameter_dictionary["W_gene_PPARg_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_PPARg_gene_Trigger"] = 1.0
	control_parameter_dictionary["W_gene_PPARg_gene_CEBPa"] = 1.0
	control_parameter_dictionary["W_gene_PPARg_gene_EGR1"] = 1.0
	control_parameter_dictionary["W_gene_PPARg_gene_PU1"] = 1.0
	control_parameter_dictionary["W_gene_PPARg_gene_AP1"] = 1.0
	control_parameter_dictionary["W_gene_PU1_RNAP"] = 0.001
	control_parameter_dictionary["W_gene_PU1_gene_Trigger"] = 1.0
	control_parameter_dictionary["W_gene_PU1_gene_CEBPa"] = 1.0
	control_parameter_dictionary["W_gene_PU1_gene_PU1"] = 1.0
	control_parameter_dictionary["W_gene_PU1_gene_AP1"] = 1.0
	control_parameter_dictionary["W_gene_PU1_gene_OCT1"] = 1.0
	control_parameter_dictionary["W_gene_PU1_gene_AhR"] = 1.0
	control_parameter_dictionary["W_gene_PU1_gene_GFI1"] = 1.0

	# These need to be zero => no background expression -
	control_parameter_dictionary["W_gene_Trigger_RNAP"] = 0.0;
	control_parameter_dictionary["W_gene_cRAF_RNAP"] = 0.0;

	# Parameter name index array -
	parameter_name_mapping_array = [
		"n_gene_AP1_gene_AhR"	;	# 1
		"K_gene_AP1_gene_AhR"	;	# 2
		"n_gene_AP1_gene_PU1"	;	# 3
		"K_gene_AP1_gene_PU1"	;	# 4
		"n_gene_AP1_gene_PPARg"	;	# 5
		"K_gene_AP1_gene_PPARg"	;	# 6
		"n_gene_AhR_gene_Trigger"	;	# 7
		"K_gene_AhR_gene_Trigger"	;	# 8
		"n_gene_CD11b_gene_PU1_gene_cRAF"	;	# 9
		"K_gene_CD11b_gene_PU1_gene_cRAF"	;	# 10
		"n_gene_CD14_gene_PPARg_gene_CEBPa_gene_EGR1_gene_cRAF"	;	# 11
		"K_gene_CD14_gene_PPARg_gene_CEBPa_gene_EGR1_gene_cRAF"	;	# 12
		"n_gene_CD38_gene_IRF1_gene_PPARg_gene_Trigger_gene_cRAF"	;	# 13
		"K_gene_CD38_gene_IRF1_gene_PPARg_gene_Trigger_gene_cRAF"	;	# 14
		"n_gene_CEBPa_gene_Trigger"	;	# 15
		"K_gene_CEBPa_gene_Trigger"	;	# 16
		"n_gene_CEBPa_gene_PPARg"	;	# 17
		"K_gene_CEBPa_gene_PPARg"	;	# 18
		"n_gene_CEBPa_gene_CEBPa"	;	# 19
		"K_gene_CEBPa_gene_CEBPa"	;	# 20
		"n_gene_CEBPa_gene_GFI1"	;	# 21
		"K_gene_CEBPa_gene_GFI1"	;	# 22
		"n_gene_E2F_gene_E2F"	;	# 23
		"K_gene_E2F_gene_E2F"	;	# 24
		"n_gene_E2F_gene_PPARg"	;	# 25
		"K_gene_E2F_gene_PPARg"	;	# 26
		"n_gene_E2F_gene_CEBPa"	;	# 27
		"K_gene_E2F_gene_CEBPa"	;	# 28
		"n_gene_E2F_gene_GFI1"	;	# 29
		"K_gene_E2F_gene_GFI1"	;	# 30
		"n_gene_E2F_gene_cRAF"	;	# 31
		"K_gene_E2F_gene_cRAF"	;	# 32
		"n_gene_EGR1_gene_Trigger"	;	# 33
		"K_gene_EGR1_gene_Trigger"	;	# 34
		"n_gene_EGR1_gene_PU1"	;	# 35
		"K_gene_EGR1_gene_PU1"	;	# 36
		"n_gene_EGR1_gene_PPARg"	;	# 37
		"K_gene_EGR1_gene_PPARg"	;	# 38
		"n_gene_EGR1_gene_GFI1"	;	# 39
		"K_gene_EGR1_gene_GFI1"	;	# 40
		"n_gene_GFI1_gene_CEBPa"	;	# 41
		"K_gene_GFI1_gene_CEBPa"	;	# 42
		"n_gene_GFI1_gene_EGR1"	;	# 43
		"K_gene_GFI1_gene_EGR1"	;	# 44
		"n_gene_IRF1_gene_Trigger"	;	# 45
		"K_gene_IRF1_gene_Trigger"	;	# 46
		"n_gene_IRF1_gene_AhR"	;	# 47
		"K_gene_IRF1_gene_AhR"	;	# 48
		"n_gene_IRF1_gene_PPARg"	;	# 49
		"K_gene_IRF1_gene_PPARg"	;	# 50
		"n_gene_OCT1_gene_PPARg"	;	# 51
		"K_gene_OCT1_gene_PPARg"	;	# 52
		"n_gene_OCT4_gene_Trigger"	;	# 53
		"K_gene_OCT4_gene_Trigger"	;	# 54
		"n_gene_OCT4_gene_AhR"	;	# 55
		"K_gene_OCT4_gene_AhR"	;	# 56
		"n_gene_OCT4_gene_cRAF"	;	# 57
		"K_gene_OCT4_gene_cRAF"	;	# 58
		"n_gene_P21_gene_Trigger_gene_AP1_gene_PPARg_gene_PU1_gene_IRF1_gene_CEBPa_gene_cRAF"	;	# 59
		"K_gene_P21_gene_Trigger_gene_AP1_gene_PPARg_gene_PU1_gene_IRF1_gene_CEBPa_gene_cRAF"	;	# 60
		"n_gene_P21_gene_GFI1"	;	# 61
		"K_gene_P21_gene_GFI1"	;	# 62
		"n_gene_P47Phox_gene_PU1_gene_CEBPa_gene_cRAF"	;	# 63
		"K_gene_P47Phox_gene_PU1_gene_CEBPa_gene_cRAF"	;	# 64
		"n_gene_P47Phox_gene_PPARg"	;	# 65
		"K_gene_P47Phox_gene_PPARg"	;	# 66
		"n_gene_PPARg_gene_Trigger"	;	# 67
		"K_gene_PPARg_gene_Trigger"	;	# 68
		"n_gene_PPARg_gene_CEBPa"	;	# 69
		"K_gene_PPARg_gene_CEBPa"	;	# 70
		"n_gene_PPARg_gene_EGR1"	;	# 71
		"K_gene_PPARg_gene_EGR1"	;	# 72
		"n_gene_PPARg_gene_PU1"	;	# 73
		"K_gene_PPARg_gene_PU1"	;	# 74
		"n_gene_PPARg_gene_AP1"	;	# 75
		"K_gene_PPARg_gene_AP1"	;	# 76
		"n_gene_PU1_gene_Trigger"	;	# 77
		"K_gene_PU1_gene_Trigger"	;	# 78
		"n_gene_PU1_gene_CEBPa"	;	# 79
		"K_gene_PU1_gene_CEBPa"	;	# 80
		"n_gene_PU1_gene_PU1"	;	# 81
		"K_gene_PU1_gene_PU1"	;	# 82
		"n_gene_PU1_gene_AP1"	;	# 83
		"K_gene_PU1_gene_AP1"	;	# 84
		"n_gene_PU1_gene_OCT1"	;	# 85
		"K_gene_PU1_gene_OCT1"	;	# 86
		"n_gene_PU1_gene_AhR"	;	# 87
		"K_gene_PU1_gene_AhR"	;	# 88
		"n_gene_PU1_gene_GFI1"	;	# 89
		"K_gene_PU1_gene_GFI1"	;	# 90
		"W_gene_AP1_RNAP"	;	# 91
		"W_gene_AP1_gene_AhR"	;	# 92
		"W_gene_AP1_gene_PU1"	;	# 93
		"W_gene_AP1_gene_PPARg"	;	# 94
		"W_gene_AhR_RNAP"	;	# 95
		"W_gene_AhR_gene_Trigger"	;	# 96
		"W_gene_CD11b_RNAP"	;	# 97
		"W_gene_CD11b_gene_PU1_gene_cRAF"	;	# 98
		"W_gene_CD14_RNAP"	;	# 99
		"W_gene_CD14_gene_PPARg_gene_CEBPa_gene_EGR1_gene_cRAF"	;	# 100
		"W_gene_CD38_RNAP"	;	# 101
		"W_gene_CD38_gene_IRF1_gene_PPARg_gene_Trigger_gene_cRAF"	;	# 102
		"W_gene_CEBPa_RNAP"	;	# 103
		"W_gene_CEBPa_gene_Trigger"	;	# 104
		"W_gene_CEBPa_gene_PPARg"	;	# 105
		"W_gene_CEBPa_gene_CEBPa"	;	# 106
		"W_gene_CEBPa_gene_GFI1"	;	# 107
		"W_gene_E2F_RNAP"	;	# 108
		"W_gene_E2F_gene_E2F"	;	# 109
		"W_gene_E2F_gene_PPARg"	;	# 110
		"W_gene_E2F_gene_CEBPa"	;	# 111
		"W_gene_E2F_gene_GFI1"	;	# 112
		"W_gene_E2F_gene_cRAF"	;	# 113
		"W_gene_EGR1_RNAP"	;	# 114
		"W_gene_EGR1_gene_Trigger"	;	# 115
		"W_gene_EGR1_gene_PU1"	;	# 116
		"W_gene_EGR1_gene_PPARg"	;	# 117
		"W_gene_EGR1_gene_GFI1"	;	# 118
		"W_gene_GFI1_RNAP"	;	# 119
		"W_gene_GFI1_gene_CEBPa"	;	# 120
		"W_gene_GFI1_gene_EGR1"	;	# 121
		"W_gene_IRF1_RNAP"	;	# 122
		"W_gene_IRF1_gene_Trigger"	;	# 123
		"W_gene_IRF1_gene_AhR"	;	# 124
		"W_gene_IRF1_gene_PPARg"	;	# 125
		"W_gene_OCT1_RNAP"	;	# 126
		"W_gene_OCT1_gene_PPARg"	;	# 127
		"W_gene_OCT4_RNAP"	;	# 128
		"W_gene_OCT4_gene_Trigger"	;	# 129
		"W_gene_OCT4_gene_AhR"	;	# 130
		"W_gene_OCT4_gene_cRAF"	;	# 131
		"W_gene_P21_RNAP"	;	# 132
		"W_gene_P21_gene_Trigger_gene_AP1_gene_PPARg_gene_PU1_gene_IRF1_gene_CEBPa_gene_cRAF"	;	# 133
		"W_gene_P21_gene_GFI1"	;	# 134
		"W_gene_P47Phox_RNAP"	;	# 135
		"W_gene_P47Phox_gene_PU1_gene_CEBPa_gene_cRAF"	;	# 136
		"W_gene_P47Phox_gene_PPARg"	;	# 137
		"W_gene_PPARg_RNAP"	;	# 138
		"W_gene_PPARg_gene_Trigger"	;	# 139
		"W_gene_PPARg_gene_CEBPa"	;	# 140
		"W_gene_PPARg_gene_EGR1"	;	# 141
		"W_gene_PPARg_gene_PU1"	;	# 142
		"W_gene_PPARg_gene_AP1"	;	# 143
		"W_gene_PU1_RNAP"	;	# 144
		"W_gene_PU1_gene_Trigger"	;	# 145
		"W_gene_PU1_gene_CEBPa"	;	# 146
		"W_gene_PU1_gene_PU1"	;	# 147
		"W_gene_PU1_gene_AP1"	;	# 148
		"W_gene_PU1_gene_OCT1"	;	# 149
		"W_gene_PU1_gene_AhR"	;	# 150
		"W_gene_PU1_gene_GFI1"	;	# 151
		"W_gene_Trigger_RNAP"	;	# 152
		"W_gene_cRAF_RNAP"	;	# 153
	]

	# Background copy number - (added by JV)
	background_copy_number_dictionary = Dict{AbstractString,Float64}()
	background_copy_number_dictionary["protein_gene_AP1"] = 100.0
	background_copy_number_dictionary["protein_gene_AhR"] = 100.0
	background_copy_number_dictionary["protein_gene_CD11b"] = 100.0
	background_copy_number_dictionary["protein_gene_CD14"] = 10.0
	background_copy_number_dictionary["protein_gene_CD38"] = 100.0
	background_copy_number_dictionary["protein_gene_CEBPa"] = 100.0
	background_copy_number_dictionary["protein_gene_E2F"] = 10000.0
	background_copy_number_dictionary["protein_gene_OCT4"] = 10000.0
	background_copy_number_dictionary["protein_gene_EGR1"] = 100.0
	background_copy_number_dictionary["protein_gene_GFI1"] = 100.0
	background_copy_number_dictionary["protein_gene_OCT1"] = 100.0
	background_copy_number_dictionary["protein_gene_P21"] = 1000.0
	background_copy_number_dictionary["protein_gene_P47Phox"] = 100.0
	background_copy_number_dictionary["protein_gene_PPARg"] = 100.0
	background_copy_number_dictionary["protein_gene_PU1"] = 1000.0
	background_copy_number_dictionary["protein_gene_IRF1"] = 100.0

	# =============================== DO NOT EDIT BELOW THIS LINE ============================== #
	data_dictionary = Dict{AbstractString,Any}()
	data_dictionary["initial_condition_array"] = initial_condition_array
	data_dictionary["average_transcript_length"] = average_transcript_length
	data_dictionary["average_protein_length"] = average_protein_length
	data_dictionary["gene_coding_length_array"] = gene_coding_length_array
	data_dictionary["mRNA_coding_length_array"] = mRNA_coding_length_array
	data_dictionary["protein_coding_length_array"] = protein_coding_length_array
	data_dictionary["rnapII_concentration"] = rnapII_concentration  # muM
	data_dictionary["ribosome_concentration"] = ribosome_concentration # muM
	data_dictionary["degradation_constant_mRNA"] = degradation_constant_mRNA  # hr^-1
	data_dictionary["degradation_constant_protein"] = degradation_constant_protein  # hr^-1
	data_dictionary["kcat_transcription"] = kcat_transcription  # hr^-1
	data_dictionary["kcat_translation"] = kcat_translation  # hr^-1
	data_dictionary["maximum_specific_growth_rate"] = maximum_specific_growth_rate  # hr^-1
	data_dictionary["death_rate_constant"] = death_rate_constant
	data_dictionary["avg_gene_concentration"] = avg_gene_concentration
	data_dictionary["saturation_constant_transcription"] = saturation_transcription
	data_dictionary["saturation_constant_translation"] = saturation_translation

	data_dictionary["stoichiometric_matrix"] = stoichiometric_matrix
	data_dictionary["dilution_matrix"] = dilution_matrix
	data_dictionary["degradation_matrix"] = degradation_matrix

	data_dictionary["binding_parameter_dictionary"] = binding_parameter_dictionary
	data_dictionary["control_parameter_dictionary"] = control_parameter_dictionary
	data_dictionary["parameter_name_mapping_array"] = parameter_name_mapping_array

	# Added by JV -
	data_dictionary["background_copy_number_dictionary"] = background_copy_number_dictionary;
	data_dictionary["cell_volume"] = V;
	data_dictionary["av_number"] = av_number;
	data_dictionary["number_of_binding"] = 90;

	# =============================== DO NOT EDIT ABOVE THIS LINE ============================== #
	return data_dictionary
end


function simulate(pomcp::POMCPOWPlanner, h_node::POWTreeObsNode{B,A,O}, s::S, d) where {B,S,A,O}

    tree = h_node.tree
    h = h_node.node

    sol = pomcp.solver

    if POMDPs.isterminal(pomcp.problem, s) || d <= 0
        return (0.0,1)
    end

    if sol.enable_action_pw
        total_n = tree.total_n[h]
        if length(tree.tried[h]) <= sol.k_action*total_n^sol.alpha_action
            if h == 1
                a = next_action(pomcp.next_action, pomcp.problem, tree.root_belief, POWTreeObsNode(tree, h))
            else
                a = next_action(pomcp.next_action, pomcp.problem, StateBelief(tree.sr_beliefs[h]), POWTreeObsNode(tree, h))
            end
            if !sol.check_repeat_act || !haskey(tree.o_child_lookup, (h,a))
                push_anode!(tree, h, a,
                            init_N(pomcp.init_N, pomcp.problem, POWTreeObsNode(tree, h), a),
                            init_V(pomcp.init_V, pomcp.problem, POWTreeObsNode(tree, h), a),
                            sol.check_repeat_act)
            end
        end
    else # run through all the actions
        if isempty(tree.tried[h])
            if h == 1
                action_space_iter = POMDPs.actions(pomcp.problem, tree.root_belief)
            else
                action_space_iter = POMDPs.actions(pomcp.problem, StateBelief(tree.sr_beliefs[h]))
            end
            anode = length(tree.n)
            for a in action_space_iter
                push_anode!(tree, h, a,
                            init_N(pomcp.init_N, pomcp.problem, POWTreeObsNode(tree, h), a),
                            init_V(pomcp.init_V, pomcp.problem, POWTreeObsNode(tree, h), a),
                            false)
            end
        end
    end
    total_n = tree.total_n[h]

    best_node = select_best(pomcp.criterion, h_node, pomcp.solver.rng)
    a = tree.a_labels[best_node]

    new_node = false
    total_counts=0
    total_reward=0.0
    flag_csyes=false
    if tree.n_a_children[best_node] <= sol.k_observation*(tree.n[best_node]^sol.alpha_observation)

        sp, o, r = @gen(:sp, :o, :r)(pomcp.problem, s, a, sol.rng)

        if sol.check_repeat_obs && haskey(tree.a_child_lookup, (best_node,o))
            hao = tree.a_child_lookup[(best_node, o)]
        else
            new_node = true
            hao = length(tree.sr_beliefs) + 1
            push!(tree.sr_beliefs,
                  init_node_sr_belief(pomcp.node_sr_belief_updater,
                                      pomcp.problem, s, a, sp, o, r))
            """
            TODO: init_node是当o为新节点时，在创建新节点的同时把s'和w(s')加入到o中。
            因此在这我们需要进行补偿性采样，把s'和weight加入到其他的所有o中，然后在其他o中也进行补偿性采样。
            """
            """
            Step 1:把s'和weight加入到其他的所有o中.
            """
            """
            Step 2:从任意一个其他o中，取出所有的(s',r)对中的s'，计算w，加入到现在的o的belief中
            """
            if(length(tree.generated_ltc[best_node])!=0)#如果存在其他o，执行step1.2d<85 &&
                first_pair=first(tree.generated_ltc[best_node])
                first_pair_hao=first_pair.second
                if(length(tree.sr_beliefs[first_pair_hao].dist.items)<100) #step1
                    for pair in tree.generated_ltc[best_node]
                        o_temp=pair.first
                        hao_temp=pair.second
                        push_weighted!(tree.sr_beliefs[hao_temp], pomcp.node_sr_belief_updater, s, sp, r)
                    end
                end
            #    step2
                for i in 1:length(tree.sr_beliefs[first_pair_hao].dist.items)#step2
                    if(i>100)
                        break
                    end
                    sp_temp=tree.sr_beliefs[first_pair_hao].dist.items[i][1]
                    r_temp=tree.sr_beliefs[first_pair_hao].dist.items[i][2]
                    push_weighted!(tree.sr_beliefs[hao], pomcp.node_sr_belief_updater, s, sp_temp, r_temp)
                end
            end

            push!(tree.total_n, 0)
            push!(tree.tried, Int[])
            push!(tree.o_labels, o)

            if sol.check_repeat_obs
                tree.a_child_lookup[(best_node, o)] = hao
            end
            tree.n_a_children[best_node] += 1
            push!(tree.generated_ltc[best_node], o=>hao)
        end
        push!(tree.generated[best_node], o=>hao)
            """
            Step3:
            对其他每个o进行补偿性采样，
            """
            # if d<90
            #     if new_node
            #         for j in 1:(length(tree.generated_ltc[best_node])-1)
            #             pair=tree.generated_ltc[best_node][j]
            #             o_temp=pair.first
            #             hao_temp=pair.second
            #             before_weight=tree.sr_beliefs[hao_temp].dist.cdf[end-1]
            #             before_sample_times=tree.total_n[hao_temp]
            #             new_added_weight=tree.sr_beliefs[hao_temp].dist.cdf[end]-before_weight
            #             cs_times=div(before_sample_times*new_added_weight,before_weight)#下取整
            #             if (cs_times!=0)
            #                 for i in 1:cs_times
            #                     R_temp,counts_temp=simulate_ow(pomcp, POWTreeObsNode(tree, hao_temp), sp, d-1)
            #                     R = r + POMDPs.discount(pomcp.problem)*R_temp
            #                     total_counts+=counts_temp
            #                     total_reward+=R*counts_temp
            #                 end
            #             end
            #         end
            #     end
            # end
    else

        sp, r = @gen(:sp, :r)(pomcp.problem, s, a, sol.rng)

    end

    if r == Inf
        @warn("POMCPOW: +Inf reward. This is not recommended and may cause future errors.")
    end

    if new_node
        R = r + POMDPs.discount(pomcp.problem)*estimate_value(pomcp.solved_estimate, pomcp.problem, sp, POWTreeObsNode(tree, hao), d-1)
        total_counts+=1
        total_reward+=R
        """
        total_counts++
        total_reward+=R
        """
    else
        pair = rand(sol.rng, tree.generated[best_node])
        o = pair.first
        hao = pair.second
        """
        To-do: 当o不是新节点时，需要把这轮从G中得到的s'，加入到当前a下的所有o分支的B(hao)中，同时根据每个分支的o计算相应的权重
        然后在每个O分支下，对新粒子进行补偿性采样。
        """
        """
        Step 1:把s'和weight加入到所有o中,包括自己.
        """
        if(length(tree.sr_beliefs[hao].dist.items)<100) #限制粒子数为1000
            for pair in tree.generated_ltc[best_node]
                o_temp=pair.first
                hao_temp=pair.second
                push_weighted!(tree.sr_beliefs[hao_temp], pomcp.node_sr_belief_updater, s, sp, r)
            end
            """
            Step2:补偿性采样
            """
            #  for pair in tree.generated_ltc[best_node]
            #      o_temp=pair.first
            #      hao_temp=pair.second
            #      before_weight=tree.sr_beliefs[hao_temp].dist.cdf[end-1]
            #      before_sample_times=tree.total_n[hao_temp]
            #      new_added_weight=tree.sr_beliefs[hao_temp].dist.cdf[end]-tree.sr_beliefs[hao_temp].dist.cdf[end-1]
            #      cs_times=div(before_sample_times*new_added_weight,before_weight)#下取整
            #      if (cs_times!=0)
            #          for i in 1:cs_times
            #              R_temp,counts_temp=simulate(pomcp, POWTreeObsNode(tree, hao_temp), sp, d-1)
            #              R = r + POMDPs.discount(pomcp.problem)*R_temp
            #              total_reward+=R*counts_temp
            #              total_counts+=counts_temp
            #          end
            #      end
            #  end
        else
            push_weighted!(tree.sr_beliefs[hao], pomcp.node_sr_belief_updater, s, sp, r)#默认把一个粒子加入到当前o
        end
      #  push_weighted!(tree.sr_beliefs[hao], pomcp.node_sr_belief_updater, s, sp, r)#默认把一个粒子加入到当前o
        temp_lenght=length(tree.sr_beliefs[hao].dist.items)
        nums_v=length(tree.total_n[hao])
        if(d<90&&nums_v>1&&nums_v<10)#temp_lenght<10) #d<30
            before_weight=tree.sr_beliefs[hao].dist.cdf[temp_lenght-1]
            before_sample_times=tree.total_n[hao]
            new_added_weight=tree.sr_beliefs[hao].dist.cdf[temp_lenght]-before_weight
            cs_times=div(before_sample_times*new_added_weight,before_weight)#下取整
            if (cs_times!=0)
                flag_csyes=true
                if(cs_times>3)
                    cs_times=2
                end
                for i in 1:cs_times
                  #  sp, r = rand(sol.rng, tree.sr_beliefs[hao])
                    R_temp,counts_temp=simulate(pomcp, POWTreeObsNode(tree, hao), sp, d-1)
                    R = r + POMDPs.discount(pomcp.problem)*R_temp
                    total_counts+=counts_temp
                    total_reward+=R*counts_temp
                end
            end
        end

       """
       Step3:随机采样

       """
       sp, r = rand(sol.rng, tree.sr_beliefs[hao])
       if(flag_csyes)
       R_temp,counts_temp=simulate(pomcp, POWTreeObsNode(tree, hao), sp, d-1)
       else
       R_temp,counts_temp=simulate(pomcp, POWTreeObsNode(tree, hao), sp, d-1)
       end

       R = r + POMDPs.discount(pomcp.problem)*R_temp
       total_counts+=counts_temp
       total_reward+=R*counts_temp
        # sp, r = rand(sol.rng, tree.sr_beliefs[hao])
        # R = r + POMDPs.discount(pomcp.problem)*simulate(pomcp, POWTreeObsNode(tree, hao), sp, d-1)
    end

    """
    返回Reward和次数
    """
    tree.n[best_node] += total_counts
    tree.total_n[h] += total_counts
    if tree.v[best_node] != -Inf
        tree.v[best_node] = ((tree.n[best_node]-total_counts)*tree.v[best_node]+total_reward)/tree.n[best_node]
    end

    return (total_reward/total_counts,total_counts)

    # tree.n[best_node] += 1
    # tree.total_n[h] += 1
    # if tree.v[best_node] != -Inf
    #     tree.v[best_node] += (R-tree.v[best_node])/tree.n[best_node]
    # end
    # return R
end

function simulate_ow(pomcp::POMCPOWPlanner, h_node::POWTreeObsNode{B,A,O}, s::S, d) where {B,S,A,O}

    tree = h_node.tree
    h = h_node.node

    sol = pomcp.solver

    if POMDPs.isterminal(pomcp.problem, s) || d <= 0
        return (0.0,1)
    end

    if sol.enable_action_pw
        total_n = tree.total_n[h]
        if length(tree.tried[h]) <= sol.k_action*total_n^sol.alpha_action
            if h == 1
                a = next_action(pomcp.next_action, pomcp.problem, tree.root_belief, POWTreeObsNode(tree, h))
            else
                a = next_action(pomcp.next_action, pomcp.problem, StateBelief(tree.sr_beliefs[h]), POWTreeObsNode(tree, h))
            end
            if !sol.check_repeat_act || !haskey(tree.o_child_lookup, (h,a))
                push_anode!(tree, h, a,
                            init_N(pomcp.init_N, pomcp.problem, POWTreeObsNode(tree, h), a),
                            init_V(pomcp.init_V, pomcp.problem, POWTreeObsNode(tree, h), a),
                            sol.check_repeat_act)
            end
        end
    else # run through all the actions
        if isempty(tree.tried[h])
            if h == 1
                action_space_iter = POMDPs.actions(pomcp.problem, tree.root_belief)
            else
                action_space_iter = POMDPs.actions(pomcp.problem, StateBelief(tree.sr_beliefs[h]))
            end
            anode = length(tree.n)
            for a in action_space_iter
                push_anode!(tree, h, a,
                            init_N(pomcp.init_N, pomcp.problem, POWTreeObsNode(tree, h), a),
                            init_V(pomcp.init_V, pomcp.problem, POWTreeObsNode(tree, h), a),
                            false)
            end
        end
    end
    total_n = tree.total_n[h]

    best_node = select_best(pomcp.criterion, h_node, pomcp.solver.rng)
    a = tree.a_labels[best_node]

    new_node = false
    total_counts=0
    total_reward=0.0
    if tree.n_a_children[best_node] <= sol.k_observation*(tree.n[best_node]^sol.alpha_observation)

        sp, o, r = @gen(:sp, :o, :r)(pomcp.problem, s, a, sol.rng)

        if sol.check_repeat_obs && haskey(tree.a_child_lookup, (best_node,o))
            hao = tree.a_child_lookup[(best_node, o)]
        else
            new_node = true
            hao = length(tree.sr_beliefs) + 1
            push!(tree.sr_beliefs,
                  init_node_sr_belief(pomcp.node_sr_belief_updater,
                                      pomcp.problem, s, a, sp, o, r))

            push!(tree.total_n, 0)
            push!(tree.tried, Int[])
            push!(tree.o_labels, o)

            if sol.check_repeat_obs
                tree.a_child_lookup[(best_node, o)] = hao
            end
            tree.n_a_children[best_node] += 1

        end
        push!(tree.generated[best_node], o=>hao)

    else

        sp, r = @gen(:sp, :r)(pomcp.problem, s, a, sol.rng)

    end

    if r == Inf
        @warn("POMCPOW: +Inf reward. This is not recommended and may cause future errors.")
    end

    if new_node
        R = r + POMDPs.discount(pomcp.problem)*estimate_value(pomcp.solved_estimate, pomcp.problem, sp, POWTreeObsNode(tree, hao), d-1)
        total_counts+=1
        total_reward+=R
        """
        total_counts++
        total_reward+=R
        """
    else
        pair = rand(sol.rng, tree.generated[best_node])
        o = pair.first
        hao = pair.second

        push_weighted!(tree.sr_beliefs[hao], pomcp.node_sr_belief_updater, s, sp, r)#默认把一个粒子加入到当前o

       """
       Step3:随机采样

       """
       sp, r = rand(sol.rng, tree.sr_beliefs[hao])

       R_temp,counts_temp=simulate_ow(pomcp, POWTreeObsNode(tree, hao), sp, d-1)

       R = r + POMDPs.discount(pomcp.problem)*R_temp
       total_counts+=counts_temp
       total_reward+=R*counts_temp
        # sp, r = rand(sol.rng, tree.sr_beliefs[hao])
        # R = r + POMDPs.discount(pomcp.problem)*simulate(pomcp, POWTreeObsNode(tree, hao), sp, d-1)
    end

    """
    返回Reward和次数
    """
    tree.n[best_node] += total_counts
    tree.total_n[h] += total_counts
    if tree.v[best_node] != -Inf
        tree.v[best_node] = ((tree.n[best_node]-total_counts)*tree.v[best_node]+total_reward)/tree.n[best_node]
    end

    return (total_reward/total_counts,total_counts)

    # tree.n[best_node] += 1
    # tree.total_n[h] += 1
    # if tree.v[best_node] != -Inf
    #     tree.v[best_node] += (R-tree.v[best_node])/tree.n[best_node]
    # end
    # return R
end


function extract_params!(pm)
    if haskey(pm, "user_defined_params")
        params = Dict{Symbol,Dict{String,Any}}()
        for key in keys(pm["user_defined_params"])
            params[Symbol(key)] = pm["user_defined_params"][key]
        end
        delete!(pm, "user_defined_params")
    end
    return params
end

function check_powermodels_data!(pm)
    if pm["correct_pm_network_data"]
        _PM.correct_network_data!(pm)
    end
    if haskey(pm, "simplify_net")
        if pm["simplify_net"]
            _PM.simplify_network!(pm)
            _PM.deactivate_isolated_components!(pm)
            _PM.propagate_topology_status!(pm)
        end
    end
    return pm
end

function active_powermodels_silence!(pm)
    if pm["silence"]
        _PM.silence()
    end
end

function check_current_limit!(pm)
    cl = 0
    for (i,branch) in pm["branch"]
        if "c_rating_a" in keys(branch)
            cl += 1
        end
    end
    return cl
end

# function calculate_current!(flows)
# # calculate branch current
# for (i, l) in flows["branch"]
#     flows["branch"][i]["if"] = abs(flows["branch"][i]["pt"] / 1 / sqrt(3))
#     flows["branch"][i]["it"] = abs(flows["branch"][i]["pf"] / 1 / sqrt(3))
#     flows["branch"][i]["i_ka"] = max.(flows["branch"][i]["if"], flows["branch"][i]["it"])
# end


using StaticArrays

# OpenVR uses something like "hungarian notation"
#   https://en.wikipedia.org/wiki/Hungarian_notation
# the german Wikipedia has more complete tables about that
#  https://de.wikipedia.org/wiki/Ungarische_Notation

# opaque pointers / forward declarations from openVR0.h:27
const VkDevice_T = Nothing
const VkPhysicalDevice_T = Nothing
const VkInstance_T = Nothing
const VkQueue_T = Nothing
const ID3D12Resource = Nothing
const ID3D12CommandQueue = Nothing

# pointer_from_objref cannot be used on immutable objects

# TODO: the SArray needs to be
#   SArray{Tuple{N₁,…,Nₙ},Int64,n,N₁*…*Nₙ}
# e.g.
#   SArray{Tuple{4},Int64,1,4}
#   SArray{Tuple{4,3},Int64,2,4*3}
#   SArray{Tuple{5,4,3},Int64,3,5*4*3}
# TODO: all these need to be immutable structs, that are _safely allocated_ when Ref is used they are not assumed to be constant anymore

# NOTE: https://en.wikipedia.org/wiki/Row-_and_column-major_order#Programming_languages_and_libraries
#  Row-major order is used in C/C++/Objective-C (for C-style arrays), PL/I, Pascal, Speakeasy, SAS, and Rasdaman.
#  Column-major order is used in Fortran, MATLAB, GNU Octave, S-Plus, R, Julia, and Scilab.

# clang-query -c="set output dump" -c="set bind-root false" -c=" match decl(eachOf(cxxRecordDecl(allOf(isExpansionInMainFile(),isStruct(),hasDefinition())).bind(\"r\"),recordDecl(allOf(isExpansionInMainFile(),isStruct(),forEach(fieldDecl().bind(\"f\")))),typedefDecl(isExpansionInMainFile()).bind(\"t\")))" -extra-arg-before=-xc++ /home/christianl/src/openvr/headers/openvr.h | sed -r "s/[[:cntrl:]]\[([0-9]{1,3};)*[0-9]{1,3}m//g" | grep -E "^CXXRecordDecl|^FieldDecl|^TypedefDecl" | sed -e "s/^FieldDecl.* \([^ ]\+\) '\([^']\+\)'.*/  \1 :: \2/" -e "s/^CXXRecordDecl.* \([^ ]\+\) definition/struct \1/" -e"s/^TypedefDecl.* \([^ ]\+\) 'union \(.\+\)'$/# const \1 = Union{\2} # NOTE: C unions are not supported in Julia/" -e "s/^TypedefDecl.* \([^ ]\+\) '\([^']\+\)'.*/const \1 = \2/" -e "s/vr:://" -e "s/float/Float32/" -e "s/double/Float64/" -e "s/char/Cchar/" -e "s/uint\([^ ]*\)_t/UInt\1/" -e "s/int\([^ ]*\)_t/Int\1/" -e "s/bool/Bool/" -e "s/\(const\)\{0,1\} *void \*/ Ptr{Nothing}/" -e "s/\(const\)\{0,1\} *\([^ ]\+\) \*/Ptr{\2}/" -e "s/ \([^ ]\+\) \[\(.*\)\]/ SArray{Tuple{\2},\1}/" -e "s/\(Tuple{.*\)\]\[\(.*}\)/\1,\2/" -e "s/\(Tuple{.*\)\]\[\(.*}\)/\1,\2/" | sed -e ":0;/^ /b1;p;z;N;s/^\n//;b0;:1;/^[^ ]/{i\\" -e "end" -e ";b0};p;z;N;s/^\n//;b1"

const SpatialAnchorHandle_t = UInt32
const glSharedTextureHandle_t = Ptr{Nothing}
const glInt_t = Int32
const glUInt_t = UInt32
struct HmdMatrix34_t
  m :: SArray{Tuple{4,3},Float32,2,3*4} # NOTE: C is row-major, Julia is column-major
end
struct HmdMatrix33_t
  m :: SArray{Tuple{3,3},Float32,2,3*3} # NOTE: C is row-major, Julia is column-major
end
struct HmdMatrix44_t
  m :: SArray{Tuple{4,4},Float32,2,4*4} # NOTE: C is row-major, Julia is column-major
end
struct HmdVector3_t
  v :: SArray{Tuple{3},Float32,1,3}
end
struct HmdVector4_t
  v :: SArray{Tuple{4},Float32,1,4}
end
struct HmdVector3d_t
  v :: SArray{Tuple{3},Float64,1,3}
end
struct HmdVector2_t
  v :: SArray{Tuple{2},Float32,1,2}
end
struct HmdQuaternion_t
  w :: Float64
  x :: Float64
  y :: Float64
  z :: Float64
end
struct HmdQuaternionf_t
  w :: Float32
  x :: Float32
  y :: Float32
  z :: Float32
end
struct HmdColor_t
  r :: Float32
  g :: Float32
  b :: Float32
  a :: Float32
end
struct HmdQuad_t
  vCorners :: SArray{Tuple{4},HmdVector3_t,1,4}
end
struct HmdRect2_t
  vTopLeft :: HmdVector2_t
  vBottomRight :: HmdVector2_t
end
struct DistortionCoordinates_t
  rfRed :: SArray{Tuple{2},Float32,1,2}
  rfGreen :: SArray{Tuple{2},Float32,1,2}
  rfBlue :: SArray{Tuple{2},Float32,1,2}
end
struct Texture_t
  handle :: Ptr{Nothing}
  eType :: ETextureType
  eColorSpace :: EColorSpace
end
const SharedTextureHandle_t = UInt64
const DriverId_t = UInt32
const TrackedDeviceIndex_t = UInt32
struct TrackedDevicePose_t
  mDeviceToAbsoluteTracking :: HmdMatrix34_t
  vVelocity :: HmdVector3_t
  vAngularVelocity :: HmdVector3_t
  eTrackingResult :: ETrackingResult
  bPoseIsValid :: Bool
  bDeviceIsConnected :: Bool
end
const WebConsoleHandle_t = UInt64
const PropertyContainerHandle_t = UInt64
const PropertyTypeTag_t = UInt32
const DriverHandle_t = PropertyContainerHandle_t
const VRActionHandle_t = UInt64
const VRActionSetHandle_t = UInt64
const VRInputValueHandle_t = UInt64
struct VRTextureBounds_t
  uMin :: Float32
  vMin :: Float32
  uMax :: Float32
  vMax :: Float32
end
struct VRTextureWithPose_t
  mDeviceToAbsoluteTracking :: HmdMatrix34_t
end
struct VRTextureDepthInfo_t
  handle :: Ptr{Nothing}
  mProjection :: HmdMatrix44_t
  vRange :: HmdVector2_t
end
struct VRTextureWithDepth_t
  depth :: VRTextureDepthInfo_t
end
struct VRTextureWithPoseAndDepth_t
  depth :: VRTextureDepthInfo_t
end
struct VRVulkanTextureData_t
  m_nImage :: UInt64
  m_pDevice ::Ptr{VkDevice_T}
  m_pPhysicalDevice ::Ptr{VkPhysicalDevice_T}
  m_pInstance ::Ptr{VkInstance_T}
  m_pQueue ::Ptr{VkQueue_T}
  m_nQueueFamilyIndex :: UInt32
  m_nWidth :: UInt32
  m_nHeight :: UInt32
  m_nFormat :: UInt32
  m_nSampleCount :: UInt32
end
struct D3D12TextureData_t
  m_pResource ::Ptr{ID3D12Resource}
  m_pCommandQueue ::Ptr{ID3D12CommandQueue}
  m_nNodeMask :: UInt32
end
struct VREvent_Controller_t
  button :: UInt32
end
struct VREvent_Mouse_t
  x :: Float32
  y :: Float32
  button :: UInt32
end
struct VREvent_Scroll_t
  xdelta :: Float32
  ydelta :: Float32
  repeatCount :: UInt32
end
struct VREvent_TouchPadMove_t
  bFingerDown :: Bool
  flSecondsFingerDown :: Float32
  fValueXFirst :: Float32
  fValueYFirst :: Float32
  fValueXRaw :: Float32
  fValueYRaw :: Float32
end
struct VREvent_Notification_t
  ulUserValue :: UInt64
  notificationId :: UInt32
end
struct VREvent_Process_t
  pid :: UInt32
  oldPid :: UInt32
  bForced :: Bool
  bConnectionLost :: Bool
end
struct VREvent_Overlay_t
  overlayHandle :: UInt64
  devicePath :: UInt64
end
struct VREvent_Status_t
  statusState :: UInt32
end
struct VREvent_Keyboard_t
  cNewInput :: SArray{Tuple{8},Cchar,1,8}
  uUserValue :: UInt64
end
struct VREvent_Ipd_t
  ipdMeters :: Float32
end
struct VREvent_Chaperone_t
  m_nPreviousUniverse :: UInt64
  m_nCurrentUniverse :: UInt64
end
struct VREvent_Reserved_t
  reserved0 :: UInt64
  reserved1 :: UInt64
  reserved2 :: UInt64
  reserved3 :: UInt64
  reserved4 :: UInt64
  reserved5 :: UInt64
end
struct VREvent_PerformanceTest_t
  m_nFidelityLevel :: UInt32
end
struct VREvent_SeatedZeroPoseReset_t
  bResetBySystemMenu :: Bool
end
struct VREvent_Screenshot_t
  handle :: UInt32
  type :: UInt32
end
struct VREvent_ScreenshotProgress_t
  progress :: Float32
end
struct VREvent_ApplicationLaunch_t
  pid :: UInt32
  unArgsHandle :: UInt32
end
struct VREvent_EditingCameraSurface_t
  overlayHandle :: UInt64
  nVisualMode :: UInt32
end
struct VREvent_MessageOverlay_t
  unVRMessageOverlayResponse :: UInt32
end
struct VREvent_Property_t
  container :: PropertyContainerHandle_t
  prop :: ETrackedDeviceProperty
end
struct VREvent_DualAnalog_t
  x :: Float32
  y :: Float32
  transformedX :: Float32
  transformedY :: Float32
  which :: EDualAnalogWhich
end
struct VREvent_HapticVibration_t
  containerHandle :: UInt64
  componentHandle :: UInt64
  fDurationSeconds :: Float32
  fFrequency :: Float32
  fAmplitude :: Float32
end
struct VREvent_WebConsole_t
  webConsoleHandle :: WebConsoleHandle_t
end
struct VREvent_InputBindingLoad_t
  ulAppContainer :: PropertyContainerHandle_t
  pathMessage :: UInt64
  pathUrl :: UInt64
  pathControllerType :: UInt64
end
struct VREvent_InputActionManifestLoad_t
  pathAppKey :: UInt64
  pathMessage :: UInt64
  pathMessageParam :: UInt64
  pathManifestPath :: UInt64
end
struct VREvent_SpatialAnchor_t
  unHandle :: SpatialAnchorHandle_t
end
struct VREvent_ProgressUpdate_t
  ulApplicationPropertyContainer :: UInt64
  pathDevice :: UInt64
  pathInputSource :: UInt64
  pathProgressAction :: UInt64
  pathIcon :: UInt64
  fProgress :: Float32
end
struct VREvent_ShowUI_t
  eType :: EShowUIType
end
# const VREvent_Data_t = Union{VREvent_Data_t':'VREvent_Data_t} # NOTE: C unions are not supported in Julia
struct VREvent_Data_t
  data :: NTuple{Int64(VREvent_Data_tLen),UInt8}
end
struct VREvent_t
  eventType :: UInt32
  trackedDeviceIndex :: TrackedDeviceIndex_t
  eventAgeSeconds :: Float32
  data :: VREvent_Data_t
end
struct HiddenAreaMesh_t
  pVertexData :: Ptr{HmdVector2_t}
  unTriangleCount :: UInt32
end
struct VRControllerAxis_t
  x :: Float32
  y :: Float32
end
struct VRControllerState001_t
  unPacketNum :: UInt32
  ulButtonPressed :: UInt64
  ulButtonTouched :: UInt64
  rAxis :: SArray{Tuple{5},VRControllerAxis_t,1,5}
end
const VRControllerState_t = VRControllerState001_t
struct Compositor_OverlaySettings
  size :: UInt32
  curved :: Bool
  antialias :: Bool
  scale :: Float32
  distance :: Float32
  alpha :: Float32
  uOffset :: Float32
  vOffset :: Float32
  uScale :: Float32
  vScale :: Float32
  gridDivs :: Float32
  gridWidth :: Float32
  gridScale :: Float32
  transform :: HmdMatrix44_t
end
const VROverlayHandle_t = UInt64
struct VRBoneTransform_t
  position :: HmdVector4_t
  orientation :: HmdQuaternionf_t
end
const BoneIndex_t = Int32
const TrackedCameraHandle_t = UInt64
struct CameraVideoStreamFrameHeader_t
  eFrameType :: EVRTrackedCameraFrameType
  nWidth :: UInt32
  nHeight :: UInt32
  nBytesPerPixel :: UInt32
  nFrameSequence :: UInt32
  standingTrackedDevicePose :: TrackedDevicePose_t
  ulFrameExposureTime :: UInt64
end
const ScreenshotHandle_t = UInt32
struct DriverDirectMode_FrameTiming
  m_nSize :: UInt32
  m_nNumFramePresents :: UInt32
  m_nNumMisPresented :: UInt32
  m_nNumDroppedFrames :: UInt32
  m_nReprojectionFlags :: UInt32
end
struct ImuSample_t
  fSampleTime :: Float64
  vAccel :: HmdVector3d_t
  vGyro :: HmdVector3d_t
  unOffScaleFlags :: UInt32
end
struct AppOverrideKeys_t
  pchKey :: Ptr{Cchar}
  pchValue :: Ptr{Cchar}
end
struct Compositor_FrameTiming
  m_nSize :: UInt32
  m_nFrameIndex :: UInt32
  m_nNumFramePresents :: UInt32
  m_nNumMisPresented :: UInt32
  m_nNumDroppedFrames :: UInt32
  m_nReprojectionFlags :: UInt32
  m_flSystemTimeInSeconds :: Float64
  m_flPreSubmitGpuMs :: Float32
  m_flPostSubmitGpuMs :: Float32
  m_flTotalRenderGpuMs :: Float32
  m_flCompositorRenderGpuMs :: Float32
  m_flCompositorRenderCpuMs :: Float32
  m_flCompositorIdleCpuMs :: Float32
  m_flClientFrameIntervalMs :: Float32
  m_flPresentCallCpuMs :: Float32
  m_flWaitForPresentCpuMs :: Float32
  m_flSubmitFrameMs :: Float32
  m_flWaitGetPosesCalledMs :: Float32
  m_flNewPosesReadyMs :: Float32
  m_flNewFrameReadyMs :: Float32
  m_flCompositorUpdateStartMs :: Float32
  m_flCompositorUpdateEndMs :: Float32
  m_flCompositorRenderStartMs :: Float32
  m_HmdPose :: TrackedDevicePose_t
  m_nNumVSyncsReadyForUse :: UInt32
  m_nNumVSyncsToFirstView :: UInt32
end
struct Compositor_CumulativeStats
  m_nPid :: UInt32
  m_nNumFramePresents :: UInt32
  m_nNumDroppedFrames :: UInt32
  m_nNumReprojectedFrames :: UInt32
  m_nNumFramePresentsOnStartup :: UInt32
  m_nNumDroppedFramesOnStartup :: UInt32
  m_nNumReprojectedFramesOnStartup :: UInt32
  m_nNumLoading :: UInt32
  m_nNumFramePresentsLoading :: UInt32
  m_nNumDroppedFramesLoading :: UInt32
  m_nNumReprojectedFramesLoading :: UInt32
  m_nNumTimedOut :: UInt32
  m_nNumFramePresentsTimedOut :: UInt32
  m_nNumDroppedFramesTimedOut :: UInt32
  m_nNumReprojectedFramesTimedOut :: UInt32
end
struct NotificationBitmap_t
  m_pImageData :: Ptr{Nothing}
  m_nWidth :: Int32
  m_nHeight :: Int32
  m_nBytesPerPixel :: Int32
end
const VRNotificationId = UInt32
struct VROverlayIntersectionParams_t
  vSource :: HmdVector3_t
  vDirection :: HmdVector3_t
  eOrigin :: ETrackingUniverseOrigin
end
struct VROverlayIntersectionResults_t
  vPoint :: HmdVector3_t
  vNormal :: HmdVector3_t
  vUVs :: HmdVector2_t
  fDistance :: Float32
end
struct IntersectionMaskRectangle_t
  m_flTopLeftX :: Float32
  m_flTopLeftY :: Float32
  m_flWidth :: Float32
  m_flHeight :: Float32
end
struct IntersectionMaskCircle_t
  m_flCenterX :: Float32
  m_flCenterY :: Float32
  m_flRadius :: Float32
end
# const VROverlayIntersectionMaskPrimitive_Data_t = Union{VROverlayIntersectionMaskPrimitive_Data_t':'VROverlayIntersectionMaskPrimitive_Data_t} # NOTE: C unions are not supported in Julia
struct VROverlayIntersectionMaskPrimitive_Data_t
  data :: NTuple{Int64(VROverlayIntersectionMaskPrimitive_Data_tLen),UInt8}
end
struct VROverlayIntersectionMaskPrimitive_t
  m_nPrimitiveType :: EVROverlayIntersectionMaskPrimitiveType
  m_Primitive :: VROverlayIntersectionMaskPrimitive_Data_t
end
const VRComponentProperties = UInt32
struct RenderModel_ComponentState_t
  mTrackingToComponentRenderModel :: HmdMatrix34_t
  mTrackingToComponentLocal :: HmdMatrix34_t
  uProperties :: VRComponentProperties
end
struct RenderModel_Vertex_t
  vPosition :: HmdVector3_t
  vNormal :: HmdVector3_t
  rfTextureCoord :: SArray{Tuple{2},Float32,1,2}
end
struct RenderModel_TextureMap_t
  unWidth :: UInt16
  unHeight :: UInt16
  rubTextureMapData1 :: UInt32 # Ptr{UInt8}
  rubTextureMapData2 :: UInt32 # Ptr{UInt8}
end
const TextureID_t = Int32
struct RenderModel_t
  rVertexData :: Ptr{RenderModel_Vertex_t} # const RenderModel_Vertex_t *rVertexData; // Vertex data for the mesh
  unVertexCount :: UInt32                  # uint32_t unVertexCount;                  // Number of vertices in the vertex data
  rIndexData1 :: UInt32 # Ptr{…}
  rIndexData2 :: UInt32 # Ptr{…}                # const uint16_t *rIndexData;              // Indices into the vertex data for each triangle
  unTriangleCount :: UInt32                # uint32_t unTriangleCount;                // Number of triangles in the mesh. Index count is 3 * TriangleCount
  diffuseTextureId :: TextureID_t          # TextureID_t diffuseTextureId;
end
struct RenderModel_ControllerMode_State_t
  bScrollWheelVisible :: Bool
end
struct InputAnalogActionData_t
  bActive :: Bool
  activeOrigin :: VRInputValueHandle_t
  x :: Float32
  y :: Float32
  z :: Float32
  deltaX :: Float32
  deltaY :: Float32
  deltaZ :: Float32
  fUpdateTime :: Float32
end
struct InputDigitalActionData_t
  bActive :: Bool
  activeOrigin :: VRInputValueHandle_t
  bState :: Bool
  bChanged :: Bool
  fUpdateTime :: Float32
end
struct InputPoseActionData_t
  bActive :: Bool
  activeOrigin :: VRInputValueHandle_t
  pose :: TrackedDevicePose_t
end
struct InputSkeletalActionData_t
  bActive :: Bool
  activeOrigin :: VRInputValueHandle_t
end
struct InputOriginInfo_t
  devicePath :: VRInputValueHandle_t
  trackedDeviceIndex :: TrackedDeviceIndex_t
  rchRenderModelComponentName :: SArray{Tuple{128},Cchar,1,128}
end
struct VRActiveActionSet_t
  ulActionSet :: VRActionSetHandle_t
  ulRestrictedToDevice :: VRInputValueHandle_t
  ulSecondaryActionSet :: VRActionSetHandle_t
  unPadding :: UInt32
  nPriority :: Int32
end
struct VRSkeletalSummaryData_t
  flFingerCurl :: SArray{Tuple{5},Float32,1,5}
  flFingerSplay :: SArray{Tuple{4},Float32,1,4}
end
const IOBufferHandle_t = UInt64
struct SpatialAnchorPose_t
  mAnchorToAbsoluteTracking :: HmdMatrix34_t
end
const HmdError = EVRInitError
const Hmd_Eye = EVREye
const ColorSpace = EColorSpace
const HmdTrackingResult = ETrackingResult
const TrackedDeviceClass = ETrackedDeviceClass
const TrackingUniverseOrigin = ETrackingUniverseOrigin
const TrackedDeviceProperty = ETrackedDeviceProperty
const TrackedPropertyError = ETrackedPropertyError
const VRSubmitFlags_t = EVRSubmitFlags
const VRState_t = EVRState
const CollisionBoundsStyle_t = ECollisionBoundsStyle
const VROverlayError = EVROverlayError
const VRFirmwareError = EVRFirmwareError
const VRCompositorError = EVRCompositorError
const VRScreenshotsError = EVRScreenshotError


"Check if formula contains a predicate name."
has_pred(formula::Const, pred_names) = formula.name in pred_names
has_pred(formula::Var, pred_names) = false
has_pred(formula::Compound, pred_names) = formula.name in pred_names ||
    any(has_pred(f, pred_names) for f in formula.args)
has_pred(formula::Term, domain::Domain) =
    has_pred(formula, keys(domain.predicates))

"Check if formula contains a numeric fluent."
has_fluent(formula::Term, state::State) =
    has_pred(formula, keys(state.fluents))
has_fluent(formula::Term, domain::Domain) =
    has_pred(formula, keys(domain.functions))

"Check if formula contains an axiom reference."
has_axiom(formula::Term, domain::Domain) = length(domain.axioms) > 0 &&
    has_pred(formula, (ax.head.name for ax in domain.axioms))

"Check if formula contains a universal or existential quantifier."
has_quantifier(formula::Term) =
    has_pred(formula, Set(Symbol[:forall, :exists]))

"Get subscript from State"
get_value(state::State, key::String) = state[key]

"Check if term is in State"
has_term_in_state(domain::Domain, state::State, term::Term) =
    return state[domain, term]

flatten_goal(problem::Problem) =
    return flatten_conjs(problem.goal)

global g_preconditions = Dict{Symbol,Vector{Vector{Term}}}()

function cache_global_preconditions(domain::Domain)
    for (act_name, act_def) in domain.actions
        conds = filter_negative_preconds(act_def)
        g_preconditions[act_name] = conds
    end
end

"Filter out negative preconditions"
function filter_negative_preconds(action_def::Action)
    conds = get_preconditions(action_def; converter=to_dnf)
    conds = [c.args for c in conds.args]
    for c in conds filter!(t -> t.name != :not, c) end
    return conds
end

"Compute axioms without negative literals"
function compute_hsp_axioms(domain::Domain)
    axioms = regularize_clauses(domain.axioms) # Regularize domain axioms
    axioms = [Clause(ax.head, [t for t in ax.body if t.name != :not])
              for ax in axioms] # Remove negative literals
    domain.axioms = Clause[]
    return domain, axioms
end

"Create state class"
function create_stat(terms::Vector{<:Term}, types::Vector{<:Term}=Term[])
    return State(types, facts, Dict{Symbol, Any}())
end

"Compute costs of one-step derivation of domain axioms"
function compute_costs_one_step_derivation(facts::Set{Term}, fact_costs::Dict{Any,Any}, ax::Clause, heur::String)
    facts_costs = convert_dict_any_to_term(fact_costs)
    _, subst = resolve(ax.body, [Clause(f, []) for f in facts])
    for s in subst
        body = [substitute(t, s) for t in ax.body]
        if heur == "delete_relaxation/h_add"
            cost = sum([0; [get(fact_costs, f, 0) for f in body]])
        else
            cost = maximum([0; [get(fact_costs, f, 0) for f in body]])
        end
        derived = substitute(ax.head, s)
        if cost < get(fact_costs, derived, Inf)
            fact_costs[derived] = cost end
    end
    return fact_costs
end

"Compute costs of all effects of available actions"
function compute_cost_action_effect(fact_costs::Dict{Any,Any}, act::Term, domain::Domain, additions::Dict{Any, Any}, heur::String)
    act_args = domain.actions[act.name].args
    subst = Subst(var => val for (var, val) in zip(act_args, act.args))
    # Look-up preconds and substitute vars
    conds = g_preconditions[act.name]
    conds = [[substitute(t, subst) for t in c] for c in conds]
    # Compute cost of reaching each action
    if heur == "delete_relaxation/h_add"
        cost = minimum([[sum([0; [get(fact_costs, f, 0) for f in conj]]) for conj in conds]; Inf])
    else
        cost = minimum([[maximum([0; [get(fact_costs, f, 0) for f in conj]]) for conj in conds]; Inf])
    end
    # Compute cost of reaching each added fact
    added = [substitute(a, subst) for a in additions[string(act.name)]]
    cost = cost + get_arbitrary_cost(act, domain) # returns 1 if arbitrary cost isn't found (only total-cost key is checked)
    for fact in added
        if cost < get(fact_costs, fact, Inf)
            fact_costs[fact] = cost end
    end
    return fact_costs
end

"Get arbitrary cost of action term"
function get_arbitrary_cost(act::Term, domain::Domain)
    effects = domain.actions[act.name].effect
    for e in effects.args
        if length(e.args) == 0
            continue
        end
        if string(e.args[1]) == "total-cost"
            if string(e.name) == "increase"
                return parse(Float64, string(e.args[2])) * 1
            elseif string(e.name) == "decrease"
                return parse(Float64, string(e.args[2])) * -1
            end
        end
    end
    return 1
end

"Return len of Julia Set"
function length_set(set::Set)
    return length(set)
end

"Initialize facts costs"
function convert_any_to_terms(set_any::Set{Any})
    return [parse(Term, s) for s in set_any]
end

"Initialize fact costs"
function init_facts_costs(facts::Set{Term})
    return Dict{Term,Float64}(f => 0 for f in facts)
end

"Get facts and state"
function get_facts_and_state(facts_costs::Dict{Any,Any}, types::Set{Term})
    real_facts_costs = convert_dict_any_to_term(facts_costs)
    facts = Set(keys(real_facts_costs))
    state = State(types, facts, Dict{Symbol,Any}())
    return facts, state
end

"Convert any any to term float64 dicts"
function convert_dict_any_to_term(dict::Dict{Any, Any})
    return Dict{Term,Float64}(f => c for (f, c) in dict)
end

"Convert any any to symbol vector(vector(term)) dicts"
function convert_dict_any_to_preconds(dict::Dict{Any, Any})
    return Dict{Symbol, Vector{Vector{Term}}}(Symbol(f) => c for (f, c) in dict)
end


"""
Ensures that the LP model in `state.gurobi_model` is up-to-date for the current
`node`. Depending on how things are configured, this could require us doing
some or all of: rebuilding the model from scratch, applying new branchings,
and reformulating disjunctions.
"""
function populate_lp_model!(
    state::CurrentState,
    form::DMIPFormulation,
    node::Node,
    node_result::NodeResult,
    config::AlgorithmConfig,
)
    if state.rebuild_model
        create_base_model!(state, form, node, config)
        apply_branchings!(state, node)
        formulate_disjunctions!(state, form, node, node_result, config)
    else
        # If this case we would like to reuse the same model and not rebuild
        # from scratch...
        if !state.on_a_dive
            # ...however, upon backtracking we need to reset bounds and reapply
            # (or reset) disjunctive formulations.
            remove_all_branchings!(state, form)
        end
        apply_branchings!(state, node)
        if config.formulation_tightening_strategy == TIGHTEN_AT_EACH_NODE
            error(
                "Cerberus does not currently support tightening formulations via problem modification; you must rebuild from scratch.",
            )
        end
        return nothing
    end
    return nothing
end

# NOTE: At all times, the first `num_variables(form)` variables in the model
# will correspond to those registered with `form`. The disjunctive formulaters
# may add additional continuous variables, but they must come after this chunk.
# This means that these variables will present, in the same order, in every
# node LP.
function create_base_model!(
    state::CurrentState,
    form::DMIPFormulation,
    node::Node,
    config::AlgorithmConfig,
)
    reset_formulation_state!(state)
    model = config.lp_solver_factory(state, config)::Gurobi.Optimizer
    state.gurobi_model = model
    for cvi in all_variables(form)
        l, u = get_bounds(form, cvi)
        # Cache the above updates in formulation. Even better,
        # batch add variables.
        vi, ci = MOI.add_constrained_variable(model, IN(l, u))
        attach_index!(state, vi)
        push!(state.constraint_state.base_state.var_constrs, ci)
    end
    for lt_constr in get_constraints(form, CCI{LT})
        # Invariant: constraint was normalized via MOIU.normalize_constant.
        ci = MOI.add_constraint(
            model,
            instantiate(lt_constr.f, state),
            lt_constr.s,
        )
        push!(state.constraint_state.base_state.lt_constrs, ci)
    end
    for gt_constr in get_constraints(form, CCI{GT})
        # Invariant: constraint was normalized via MOIU.normalize_constant.
        ci = MOI.add_constraint(
            model,
            instantiate(gt_constr.f, state),
            gt_constr.s,
        )
        push!(state.constraint_state.base_state.gt_constrs, ci)
    end
    for et_constr in get_constraints(form, CCI{ET})
        # Invariant: constraint was normalized via MOIU.normalize_constant.
        ci = MOI.add_constraint(
            model,
            instantiate(et_constr.f, state),
            et_constr.s,
        )
        push!(state.constraint_state.base_state.et_constrs, ci)
    end
    MOI.set(model, MOI.ObjectiveFunction{SAF}(), instantiate(form.obj, state))
    MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE)
    state.rebuild_model = false
    state.total_model_builds += 1
    return nothing
end

function formulate_disjunctions!(
    state::CurrentState,
    form::DMIPFormulation,
    node::Node,
    node_result::NodeResult,
    config::AlgorithmConfig,
)
    for (formulater, cvis) in form.disjunction_formulaters
        formulate!(
            state,
            form,
            formulater,
            # If we use a static formulation, it must be valid at the root. In
            # a bit of a hack, in this case we will just pass in an empty Node.
            if config.formulation_tightening_strategy == STATIC_FORMULATION
                Node()
            else
                node
            end,
            node_result,
            config,
        )
    end
    return nothing
end

function _unattached_bounds(cs::ConstraintState, node::Node, ::Type{LT})
    return view(
        node.lt_bounds,
        Colon()(cs.branch_state.num_lt_branches + 1, length(node.lt_bounds)),
    )
end
function _unattached_bounds(cs::ConstraintState, node::Node, ::Type{GT})
    return view(
        node.gt_bounds,
        Colon()(cs.branch_state.num_gt_branches + 1, length(node.gt_bounds)),
    )
end
function _unattached_constraints(cs::ConstraintState, node::Node, ::Type{LT})
    return view(
        node.lt_general_constrs,
        Colon()(
            length(cs.branch_state.lt_general_constrs) + 1,
            length(node.lt_general_constrs),
        ),
    )
end
function _unattached_constraints(cs::ConstraintState, node::Node, ::Type{GT})
    return view(
        node.gt_general_constrs,
        Colon()(
            length(cs.branch_state.gt_general_constrs) + 1,
            length(node.gt_general_constrs),
        ),
    )
end

function remove_all_branchings!(state::CurrentState, form::DMIPFormulation)
    model = state.gurobi_model
    cs = state.constraint_state
    for cvi in all_variables(form)
        l, u = get_bounds(form, cvi)
        ci = cs.base_state.var_constrs[index(cvi)]
        MOI.set(model, MOI.ConstraintSet(), ci, IN(l, u))
    end
    # TODO: Can potentially be smarter about not deleting all of these
    # constraints on a backtrack.
    MOI.delete(model, cs.branch_state.lt_general_constrs)
    MOI.delete(model, cs.branch_state.gt_general_constrs)
    empty!(cs.branch_state)
    return nothing
end

# We skip any bounds or constraints that are already added to the model. Note
# that we do this a bit dangerously--we merely count the number of
# bounds/constraints attached (as recorded in constraint_state). Therefore,
# this will only work on dives.
function apply_branchings!(state::CurrentState, node::Node)
    model = state.gurobi_model
    cs = state.constraint_state
    for lt_bound in _unattached_bounds(cs, node, LT)
        cvi = lt_bound.cvi
        vi = instantiate(cvi, state)
        ci = cs.base_state.var_constrs[index(cvi)]
        interval = MOI.get(model, MOI.ConstraintSet(), ci)
        new_interval = IN(interval.lower, min(lt_bound.s.upper, interval.upper))
        MOI.set(model, MOI.ConstraintSet(), ci, new_interval)
        cs.branch_state.num_lt_branches += 1
    end
    for gt_bound in _unattached_bounds(cs, node, GT)
        cvi = gt_bound.cvi
        vi = instantiate(cvi, state)
        ci = cs.base_state.var_constrs[index(cvi)]
        interval = MOI.get(model, MOI.ConstraintSet(), ci)
        new_interval = IN(max(gt_bound.s.lower, interval.lower), interval.upper)
        MOI.set(model, MOI.ConstraintSet(), ci, new_interval)
        cs.branch_state.num_gt_branches += 1
    end
    for ac in _unattached_constraints(cs, node, LT)
        # Invariant: constraint was normalized via MOIU.normalize_constant.
        ci = MOI.add_constraint(model, instantiate(ac.f, state), ac.s)
        push!(cs.branch_state.lt_general_constrs, ci)
    end
    for ac in _unattached_constraints(cs, node, GT)
        # Invariant: constraint was normalized via MOIU.normalize_constant.
        ci = MOI.add_constraint(model, instantiate(ac.f, state), ac.s)
        push!(cs.branch_state.gt_general_constrs, ci)
    end
    return nothing
end

# TODO: Unit test
function _update_lp_solution!(state::CurrentState, form::DMIPFormulation)
    # NOTE: I believe that the following is costless if the size is correct.
    resize!(state.current_solution, num_variables(form))
    for cvi in all_variables(form)
        state.current_solution[index(cvi)] = MOI.get(
            state.gurobi_model,
            MOI.VariablePrimal(),
            instantiate(cvi, state),
        )
    end
    return nothing
end

function get_basis(state::CurrentState)::Basis
    basis = Basis()
    cs = state.constraint_state
    for ci in cs.base_state.var_constrs
        push!(
            basis.base_var_constrs,
            MOI.get(state.gurobi_model, MOI.ConstraintBasisStatus(), ci),
        )
    end
    for ci in cs.base_state.lt_constrs
        push!(
            basis.base_lt_constrs,
            MOI.get(state.gurobi_model, MOI.ConstraintBasisStatus(), ci),
        )
    end
    for ci in cs.base_state.gt_constrs
        push!(
            basis.base_gt_constrs,
            MOI.get(state.gurobi_model, MOI.ConstraintBasisStatus(), ci),
        )
    end
    for ci in cs.base_state.et_constrs
        push!(
            basis.base_et_constrs,
            MOI.get(state.gurobi_model, MOI.ConstraintBasisStatus(), ci),
        )
    end
    for ci in cs.branch_state.lt_general_constrs
        push!(
            basis.branch_lt_constrs,
            MOI.get(state.gurobi_model, MOI.ConstraintBasisStatus(), ci),
        )
    end
    for ci in cs.branch_state.gt_general_constrs
        push!(
            basis.branch_gt_constrs,
            MOI.get(state.gurobi_model, MOI.ConstraintBasisStatus(), ci),
        )
    end
    return basis
end

function set_basis_if_available!(state::CurrentState, node::Node)
    if haskey(state.warm_starts, node)
        _set_basis!(
            state.gurobi_model,
            state.constraint_state,
            state.warm_starts[node],
        )
        state.total_warm_starts += 1
    end
    return nothing
end

function _set_basis!(
    model::MOI.AbstractOptimizer,
    cs::ConstraintState,
    basis::Basis,
)::Nothing
    # TODO: Check that basis is, in fact, a basis after modification
    @debug "Basis is being set ($(length(basis)) elements)"
    if isempty(basis.base_var_constrs) &&
       isempty(basis.base_lt_constrs) &&
       isempty(basis.base_gt_constrs) &&
       isempty(basis.base_et_constrs) &&
       isempty(basis.branch_lt_constrs) &&
       isempty(basis.branch_gt_constrs)
        throw(ArgumentError("You are attempting to set an empty basis."))
    end
    for (i, bs) in enumerate(basis.base_var_constrs)
        ci = cs.base_state.var_constrs[i]
        MOI.set(model, MOI.ConstraintBasisStatus(), ci, bs)
    end
    for (i, bs) in enumerate(basis.base_lt_constrs)
        ci = cs.base_state.lt_constrs[i]
        MOI.set(model, MOI.ConstraintBasisStatus(), ci, bs)
    end
    for (i, bs) in enumerate(basis.base_gt_constrs)
        ci = cs.base_state.gt_constrs[i]
        MOI.set(model, MOI.ConstraintBasisStatus(), ci, bs)
    end
    for (i, bs) in enumerate(basis.base_et_constrs)
        ci = cs.base_state.et_constrs[i]
        MOI.set(model, MOI.ConstraintBasisStatus(), ci, bs)
    end
    for (i, bs) in enumerate(basis.branch_lt_constrs)
        ci = cs.branch_state.lt_general_constrs[i]
        MOI.set(model, MOI.ConstraintBasisStatus(), ci, bs)
    end
    for (i, bs) in enumerate(basis.branch_gt_constrs)
        ci = cs.branch_state.gt_general_constrs[i]
        MOI.set(model, MOI.ConstraintBasisStatus(), ci, bs)
    end
    return nothing
end


@testset "core.jl" begin

@testset "Trainer" begin
    _trainer = Trainer(;
        policy=BasicDQNLearner(
            approximator = NeuralNetworkApproximator(
                model = Chain(
                    Dense(4, 128, relu; initW = glorot_uniform),
                    Dense(128, 128, relu; initW = glorot_uniform),
                    Dense(128, 2; initW = glorot_uniform),
                ) |> cpu,
                optimizer = ADAM(),
            ),
            loss_func = huber_loss,
        )
    )

    trainer = actor(_trainer)

    put!(trainer, FetchParamMsg())
    ps = take!(self())
    original_sum = sum(sum, ps.data)

    for x in ps.data
        fill!(x, 0.)
    end

    put!(trainer, FetchParamMsg())
    ps = take!(self())
    new_sum = sum(sum, ps.data)

    # make sure no state sharing between messages
    @test original_sum == new_sum

    batch_data = (
        state = rand(4, 32),
        action = rand(1:2, 32),
        reward = rand(32),
        terminal = rand(Bool, 32),
        next_state = rand(4,32),
        next_action = rand(1:2, 32)
    )

    put!(trainer, BatchDataMsg(batch_data))

    put!(trainer, FetchParamMsg())
    ps = take!(self())
    updated_sum = sum(sum, ps.data)
    @test original_sum != updated_sum
end

@testset "TrajectoryManager" begin
    _trajectory_proxy = TrajectoryManager(
        trajectory = CircularSARTSATrajectory(;capacity=5, state_type=Any, ),
        sampler = UniformBatchSampler(3),
        inserter = NStepInserter(),
    )

    trajectory_proxy = actor(_trajectory_proxy)

    # 1. init traj for testing
    traj = CircularCompactSARTSATrajectory(
        capacity = 2,
        state_type = Float32,
        state_size = (4,),
    )
    push!(traj;state=rand(Float32, 4), action=rand(1:2))
    push!(traj;reward=rand(), terminal=rand(Bool),state=rand(Float32, 4), action=rand(1:2))
    push!(traj;reward=rand(), terminal=rand(Bool),state=rand(Float32, 4), action=rand(1:2))

    # 2. insert
    put!(trajectory_proxy, InsertTrajectoryMsg(deepcopy(traj)))  #!!! we used deepcopy here

    # 3. make sure the above message is already been handled
    put!(trajectory_proxy, PingMsg())
    take!(self())

    # 4. test that updating traj will not affect data in trajectory_proxy
    s_tp = _trajectory_proxy.trajectory[:state]
    s_traj = traj[:state]

    @test s_tp[1] == s_traj[:, 1]

    push!(traj;reward=rand(), terminal=rand(Bool),state=rand(Float32, 4), action=rand(1:2))

    @test s_tp[1] != s_traj[:, 1]

    s = sample(_trajectory_proxy.trajectory, _trajectory_proxy.sampler)
    fill!(s[:state], 0.)
    @test any(x -> sum(x) == 0, s_tp) == false  # make sure sample create an independent copy
end

@testset "Worker" begin
    _worker = Worker() do worker_proxy
        Experiment(
            Agent(
                policy = StaticPolicy(
                        QBasedPolicy(
                        learner = BasicDQNLearner(
                            approximator = NeuralNetworkApproximator(
                                model = Chain(
                                    Dense(4, 128, relu; initW = glorot_uniform),
                                    Dense(128, 128, relu; initW = glorot_uniform),
                                    Dense(128, 2; initW = glorot_uniform),
                                ) |> cpu,
                                optimizer = ADAM(),
                            ),
                            loss_func = huber_loss,
                        ),
                        explorer = EpsilonGreedyExplorer(
                            kind = :exp,
                            ϵ_stable = 0.01,
                            decay_steps = 500,
                        ),
                    ),
                ),
                trajectory = CircularCompactSARTSATrajectory(
                    capacity = 10,
                    state_type = Float32,
                    state_size = (4,),
                ),
            ),
            CartPoleEnv(; T = Float32),
            ComposedStopCondition(
                StopAfterStep(1_000),
                StopSignal(),
            ),
            ComposedHook(
                UploadTrajectoryEveryNStep(mailbox=worker_proxy, n=10, sealer=x -> InsertTrajectoryMsg(deepcopy(x))),
                LoadParamsHook(),
                TotalRewardPerEpisode(),
            ),
            "experimenting..."
        )
    end

    worker = actor(_worker)
    tmp_mailbox = Channel(100)
    put!(worker, StartMsg(tmp_mailbox))
end

@testset "WorkerProxy" begin
    target = RemoteChannel(() -> Channel(10))
    workers = [RemoteChannel(()->Channel(10)) for _ in 1:10]
    _wp = WorkerProxy(workers)
    wp = actor(_wp)

    put!(wp, StartMsg(target))
    for w in workers
        # @test take!(w).args[1] === wp
        @test Distributed.channel_from_id(remoteref_id(take!(w).args[1])) === Distributed.channel_from_id(remoteref_id(wp))
    end

    msg = InsertTrajectoryMsg(1)
    put!(wp, msg)
    @test take!(target) === msg

    for w in workers
        put!(wp, FetchParamMsg(w))
    end
    # @test take!(target).from === wp
    @test Distributed.channel_from_id(remoteref_id(take!(target).from)) === Distributed.channel_from_id(remoteref_id(wp))

    # make sure target only received one FetchParamMsg
    msg = PingMsg()
    put!(target, msg)
    @test take!(target) === msg

    msg = LoadParamMsg([])
    put!(wp, msg)
    for w in workers
        @test take!(w) === msg
    end
end

@testset "Orchestrator" begin
    # TODO
    # Add an integration test
end

end


export solicitation_table_parser

function solicitation_table_parser(df,target_rho,total_vol)

    lbbbl_to_kgm3=2.857142857142857u"kg/m^3"

    ppg_to_kgm3=119.83u"kg/m^3"

    galbbl_to_mLmL=0.0238

    mass=Float64[]
    mass_unit=Unitful.Quantity[]

    QSP_indexes=Integer[]
    vol_indexes=Integer[]

    water_index=Integer[]
    water_content=Fluid[]
    oil_index=Integer[]
    oil_content=Fluid[]
    solids_index=Integer[]
    solid_content=Solid[]

    aditive_indexes=Integer[]

    has_QSP=false
    has_pure_water=false
    pure_water_index=0
    len_components=size(df)[1]

    #check if QSP of water and salt
    comp_df=copy(df)
    if any(df.Componente .== :H2O) && any(df.Componente .== :NaCl)
        w_i=findall(df.Componente .== :H2O)
        Na_i=findall(df.Componente .== :NaCl)

        w_QSP=any()

    end

    for i in range(1,len_components)

        row=df[i,:]

        # println(row.Componente)
        if row.unit == "lb/bbl"
            convert=lbbbl_to_kgm3
            m=upreferred(row.conc*convert*total_vol)
            if row.Componente == :Obiturante
                append!(solid_content,[CaCO3()])
                append!(solids_index,i)
            elseif row.Componente == :Reologico
                ben=Bentonite()
                append!(solid_content,[ben])
                append!(solids_index,i)
            end
        elseif row.unit == "bbl/bbl" || row.unit == "v/v"
            vol=row.conc*total_vol
            append!(vol_indexes,i)
            rho=1.0
            if row.Componente == :Olefina || row.Componente == :Hexadeceno
                fluid=Hexadecene()
                rho=0.728*1000.0
                # rho=fluid.ρ
                append!(oil_index,i)
                append!(oil_content,[fluid])
            elseif row.Componente == :Brine_NaCl && !ismissing(row.conc)
                fluid=BrineNaCl(9.8)
                rho=fluid.ρ
                append!(water_index,i)
                append!(water_content,[fluid])
            elseif row.Componente == :Obiturante
                append!(solid_content,[CaCO3()])
                rho=CaCO3().ρ
                append!(solids_index,i)
            end

            m=upreferred(vol*rho*u"kg/m^3")
        elseif row.unit == "% v/v"
            vol=row.conc*total_vol/100

            if row.Componente == :Liovac4260
                rho=700.0
            end
            println("Liovac")
            m=upreferred(vol*rho*u"kg/m^3")
        elseif row.unit == "gal/bbl"
            vol=galbbl_to_mLmL*total_vol

            if row.Componente == :Polifoan
                rho=1.01u"g/cm^3"
            end

            m=upreferred(vol*rho)
        elseif ismissing(row.conc) && occursin(row.unit,"QSP")
            has_QSP=true
            append!(QSP_indexes,i)
            m=0.0u"kg"

            if row.Componente == :Barita
                append!(solids_index,i)
                append!(solid_content,[Barite()])
            elseif row.Componente == :H2O
                has_pure_water=true
                pure_water_index=i
            end

        end
        println((i=i,))
        append!(mass_unit,m)
        append!(mass,ustrip(m))
    end
    println(mass_unit)

    ## Encontra as massas dos QSP...
    fluid_rho=target_rho*ppg_to_kgm3
    total_mass=upreferred(fluid_rho*total_vol)
    if !isempty(QSP_indexes)
        if length(QSP_indexes) == 1
            m_a=0.0u"kg"
            m_B=0.0u"kg"
            for i in range(1,len_components)
                if df[i,:].aditive
                    append!(aditive_indexes,i)
                    m_a+=mass_unit[i]
                end
                if i in vol_indexes
                    m_B+=mass_unit[i]
                end
            end
            m_s=0.0u"kg"
            for j in solids_index
                if j ∉ QSP_indexes
                    m_s+=mass_unit[j]
                end
            end
            mass_unit[QSP_indexes[1]]=total_mass-m_a-m_B
        else
            # if has_pure_water && any(findall(.Componente .== :NaCl))
            #     println("hello")
            #     @error "Not Implemented"
            # end
        end
    end

    ## Encontrar as frações volumétricas considerando aditivos dissolvidos na água.

    fP=[0.0,0.0,0.0]
    fcomp=[
        [],
        [],
        []
    ]

    vol_sol=Unitful.Quantity[]
    vol_oil=0.0u"m^3"
    vol_water=0.0u"m^3"

    for (i,f) in zip(oil_index,oil_content)
        rho=f.ρ*u"kg/m^3"
        vol=upreferred(mass_unit[i]/rho)
        vol_oil+=vol
        append!(fcomp[2],vol_oil)
    end

    for (i,s) in zip(solids_index,solid_content)
        rho=s.ρ*u"kg/m^3"
        vol=upreferred(mass_unit[i]/rho)
        append!(vol_sol,vol)
        append!(fcomp[3],vol)
    end

    vol_water=total_vol-sum(vol_oil)-sum(vol_sol)

    fP[1]=upreferred(vol_water/total_vol)
    fP[2]=upreferred(sum(vol_oil)/total_vol)
    fP[3]=upreferred(sum(vol_sol)/total_vol)

    fcomp[1] = [upreferred(vol_water/total_vol)]
    fcomp[2] .= upreferred.(fcomp[2]./total_vol)
    fcomp[3] = upreferred.(fcomp[3]./total_vol)

    mass_frac=mass_unit./total_mass

    df.mass=mass_unit
    (
        total_mass=total_mass,
        total_mass_no_unit=ustrip(total_mass),
        total_vol=total_vol,
        total_vol_no_unit=ustrip(upreferred(total_vol)),
        components_mass=mass_unit,
        components_mass_no_unit=mass,
        aditive_mass=m_a,
        mass_frac=mass_frac,
        fP=fP,
        fcomp=fcomp,
        rho_std=target_rho*ppg_to_kgm3,
        solid_indexes=solids_index,
        aditive_indexes=aditive_indexes,
        df=df,
        # sol_indexes=solids_index
    )

end