isdefined(Base, :__precompile__) && __precompile__()
module PowerSimulations

#################################################################################
# Exports

# Base Models
export Simulation
export DecisionModel
export EmulationModel
export ProblemResults
export ProblemTemplate
export InitialCondition
export SimulationModels
export SimulationSequence
export SimulationResults

# Network Relevant Exports
export NetworkModel
export StandardPTDFModel
export PTDFPowerModel
export CopperPlatePowerModel
export AreaBalancePowerModel

######## Device Models ########
export DeviceModel
export FixedOutput
######## Service Models ########
export ServiceModel
export RangeReserve
export RampReserve
export StepwiseCostReserve
export NonSpinningReserve
export PIDSmoothACE
export GroupReserve
######## Branch Models ########
export StaticBranch
export StaticBranchBounds
export StaticBranchUnbounded
export HVDCLossless
export HVDCDispatch
export HVDCUnbounded
# export VoltageSourceDC
######## Load Models ########
export StaticPowerLoad
export InterruptiblePowerLoad
export DispatchablePowerLoad
######## Renewable Formulations ########
export RenewableFullDispatch
export RenewableConstantPowerFactor
######## Hydro Formulations ########
export HydroDispatchRunOfRiver
export HydroDispatchReservoirBudget
export HydroDispatchReservoirStorage
export HydroCommitmentRunOfRiver
export HydroCommitmentReservoirBudget
export HydroCommitmentReservoirStorage
export HydroDispatchPumpedStorage

######## Renewable Formulations ########
export BookKeeping
export BatteryAncillaryServices
export EnergyTarget

######## Thermal Formulations ########
export ThermalStandardUnitCommitment
export ThermalBasicUnitCommitment
export ThermalBasicCompactUnitCommitment
export ThermalBasicDispatch
export ThermalStandardDispatch
export ThermalDispatchNoMin
export ThermalMultiStartUnitCommitment
export ThermalCompactUnitCommitment
export ThermalCompactDispatch

###### Regulation Device Formulation #######
export DeviceLimitedRegulation
export ReserveLimitedRegulation

######## Hybrid Formulations ########
export BasicHybridDispatch
export StandardHybridDispatch

# feedforward models
export UpperBoundFeedforward
export LowerBoundFeedforward
export SemiContinuousFeedforward
export EnergyLimitFeedforward
export FixValueFeedforward
export EnergyTargetFeedforward

# InitialConditions chrons
export InterProblemChronology
export IntraProblemChronology

# Initial Conditions Quantities
export DevicePower
export DeviceStatus
export InitialTimeDurationOn
export InitialTimeDurationOff
export InitialEnergyLevel

# operation_models
export GenericOpProblem
export UnitCommitmentProblem
export EconomicDispatchProblem
# export OptimalPowerFlow

# Functions
export build!
## Op Model Exports
export get_initial_conditions
export serialize_problem
export serialize_results
export serialize_optimization_model
## Decision Model Export
export solve!
## Emulation Model Exports
export run!
## Sim Model Exports
export execute!
export get_simulation_model
## Template Exports
export template_economic_dispatch
export template_unit_commitment
export template_agc_reserve_deployment
export EconomicDispatchProblem
export UnitCommitmentProblem
export AGCReserveDeployment
export run_economic_dispatch
export run_unit_commitment
export set_device_model!
export set_service_model!
export set_network_model!
export get_network_formulation
## Results interfaces
export SimulationResultsExport
export ProblemResultsExport
export export_results
export export_realized_results
export export_optimizer_stats
export get_variable_values
export get_dual_values
export get_parameter_values
export get_aux_variable_values
export get_expression_values
export get_timestamps
export get_model_name
export get_decision_problem_results
export get_emulation_problem_results
export get_system
export get_system!
export set_system!
export list_variable_keys
export list_dual_keys
export list_parameter_keys
export list_aux_variable_keys
export list_expression_keys
export list_variable_names
export list_dual_names
export list_parameter_names
export list_aux_variable_names
export list_expression_names
export list_decision_problems
export list_supported_formats
export load_results!
export read_variable
export read_dual
export read_parameter
export read_aux_variable
export read_expression
export read_variables
export read_duals
export read_parameters
export read_aux_variables
export read_expressions
export read_realized_variable
export read_realized_dual
export read_realized_parameter
export read_realized_aux_variable
export read_realized_expression
export read_realized_variables
export read_realized_duals
export read_realized_parameters
export read_realized_aux_variables
export read_realized_expressions
export get_realized_timestamps
export get_problem_base_power
export get_objective_value
export read_optimizer_stats

## Utils Exports
export get_all_constraint_index
export get_all_variable_index
export get_constraint_index
export get_variable_index
export list_recorder_events
export show_recorder_events
export list_simulation_events
export show_simulation_events
export export_realized_results

## Enums
export BuildStatus
export RunStatus

# Variables
export ActivePowerVariable
export ActivePowerInVariable
export ActivePowerOutVariable
export HotStartVariable
export WarmStartVariable
export ColdStartVariable
export EnergyVariable
export EnergyVariableUp
export EnergyVariableDown
export EnergyShortageVariable
export EnergySurplusVariable
export LiftVariable
export OnVariable
export ReactivePowerVariable
export ReservationVariable
export ActivePowerReserveVariable
export ServiceRequirementVariable
export WaterSpillageVariable
export StartVariable
export StopVariable
export SteadyStateFrequencyDeviation
export AreaMismatchVariable
export DeltaActivePowerUpVariable
export DeltaActivePowerDownVariable
export AdditionalDeltaActivePowerUpVariable
export AdditionalDeltaActivePowerDownVariable
export SmoothACE
export SystemBalanceSlackUp
export SystemBalanceSlackDown
export ReserveRequirementSlack
export VoltageMagnitude
export VoltageAngle
export FlowActivePowerVariable
export FlowActivePowerFromToVariable
export FlowActivePowerToFromVariable
export FlowReactivePowerFromToVariable
export FlowReactivePowerToFromVariable
export PowerAboveMinimumVariable

# Auxiliary variables
export TimeDurationOn
export TimeDurationOff
export PowerOutput
export EnergyOutput

# Constraints
export AbsoluteValueConstraint
export ActiveConstraint
export ActivePowerVariableLimitsConstraint
export ActivePowerVariableTimeSeriesLimitsConstraint
export ActiveRangeConstraint
export ActiveRangeICConstraint
export AreaDispatchBalanceConstraint
export AreaParticipationAssignmentConstraint
export BalanceAuxConstraint
export CommitmentConstraint
export CopperPlateBalanceConstraint
export DurationConstraint
export EnergyBalanceConstraint
export EnergyBudgetConstraint
export EnergyCapacityConstraint
export EnergyCapacityDownConstraint
export EnergyCapacityUpConstraint
export EnergyLimitConstraint
export EnergyShortageVariableLimitsConstraint
export EnergyTargetConstraint
export EqualityConstraint
export FeedforwardSemiContinousConstraint
export FeedforwardUpperBoundConstraint
export FeedforwardLowerBoundConstraint
export FeedforwardIntegralLimitConstraint
export FlowActivePowerConstraint
export FlowActivePowerFromToConstraint
export FlowActivePowerToFromConstraint
export FlowLimitConstraint
export FlowLimitFromToConstraint
export FlowLimitToFromConstraint
export FlowRateConstraint
export FlowRateConstraintFromTo
export FlowRateConstraintToFrom
export FlowReactivePowerConstraint
export FlowReactivePowerFromToConstraint
export FlowReactivePowerToFromConstraint
export FrequencyResponseConstraint
export HVDCPowerBalance
export HVDCTotalPowerDeliveredVariable
export InflowRangeConstraint
export InputActivePowerVariableLimitsConstraint
export InputPowerRangeConstraint
export MustRunConstraint
export NetworkFlowConstraint
export NodalBalanceActiveConstraint
export NodalBalanceReactiveConstraint
export OutputActivePowerVariableLimitsConstraint
export PieceWiseLinearCostConstraint
export PowerOutputRangeConstraint
export ParticipationAssignmentConstraint
export RampConstraint
export RampLimitConstraint
export RangeLimitConstraint
export RateLimitConstraint
export RateLimitConstraintFromTo
export RateLimitConstraintToFrom
export ReactivePowerVariableLimitsConstraint
export ReactiveRangeConstraint
export RegulationLimitsConstraint
export RequirementConstraint
export ReserveEnergyConstraint
export ReservePowerConstraint
export SACEPIDAreaConstraint
export StartTypeConstraint
export StartupInitialConditionConstraint
export StartupTimeLimitTemperatureConstraint

# Parameters
# Time Series Parameters
export ActivePowerTimeSeriesParameter
export ReactivePowerTimeSeriesParameter
export RequirementTimeSeriesParameter
export EnergyTargetTimeSeriesParameter
export EnergyBudgetTimeSeriesParameter

# Feedforward Parameters
export OnStatusParameter
export UpperBoundValueParameter

# Expressions
export SystemBalanceExpressions
export RangeConstraintLBExpressions
export RangeConstraintUBExpressions
export CostExpressions
export ActivePowerBalance
export ReactivePowerBalance
export EmergencyUp
export EmergencyDown
export RawACE
export ProductionCostExpression
export ActivePowerRangeExpressionLB
export ComponentActivePowerRangeExpressionLB
export ReserveRangeExpressionLB
export ActivePowerRangeExpressionUB
export ReserveRangeExpressionUB
export ComponentActivePowerRangeExpressionUB
export ComponentReserveUpBalanceExpression
export ComponentReserveDownBalanceExpression

#################################################################################
# Imports
import DataStructures: OrderedDict, Deque, SortedDict
import Logging
import Serialization
# Modeling Imports
import JuMP
# so that users do not need to import JuMP to use a solver with PowerModels
import JuMP: optimizer_with_attributes
import JuMP.Containers: DenseAxisArray, SparseAxisArray
export optimizer_with_attributes
import MathOptInterface
import ParameterJuMP
import LinearAlgebra
import JSON3
import PowerSystems
import InfrastructureSystems
import InfrastructureSystems: @assert_op, list_recorder_events, get_name
export get_name
export get_model_base_power
export get_optimizer_stats
export get_timestamps
export get_resolution
import PowerModels
import TimerOutputs
import ProgressMeter

# Base Imports
import Base.getindex
import Base.length
import Base.first

# TimeStamp Management Imports
import Dates
import TimeSeries

# I/O Imports
import DataFrames
import JSON
import CSV
import SHA
import HDF5
import PrettyTables

# PowerModels exports
export ACPPowerModel
export ACRPowerModel
export ACTPowerModel
export DCPPowerModel
export NFAPowerModel
export DCPLLPowerModel
export LPACCPowerModel
export SOCWRPowerModel
export SOCWRConicPowerModel
export QCRMPowerModel
export QCLSPowerModel

################################################################################

# Type Alias From other Packages
const PM = PowerModels
const PSY = PowerSystems
const PSI = PowerSimulations
const IS = InfrastructureSystems
const MOI = MathOptInterface
const MOIU = MathOptInterface.Utilities
const PJ = ParameterJuMP
const MOPFM = MOI.FileFormats.Model
const TS = TimeSeries

################################################################################

function progress_meter_enabled()
    return isa(stderr, Base.TTY) &&
           (get(ENV, "CI", nothing) != "true") &&
           (get(ENV, "RUNNING_PSI_TESTS", nothing) != "true")
end

using DocStringExtensions

@template DEFAULT = """
                    $(TYPEDSIGNATURES)
                    $(DOCSTRING)
                    $(METHODLIST)
                    """
# Includes

include("core/definitions.jl")

# Core components
include("core/formulations.jl")
include("core/abstract_simulation_store.jl")
include("core/operation_model_abstract_types.jl")
include("core/optimization_container_types.jl")
include("core/abstract_feedforward.jl")
include("core/optimization_container_keys.jl")
include("core/network_model.jl")
include("core/parameters.jl")
include("core/service_model.jl")
include("core/device_model.jl")
include("core/variables.jl")
include("core/auxiliary_variables.jl")
include("core/constraints.jl")
include("core/expressions.jl")
include("core/initial_conditions.jl")
include("core/settings.jl")
include("core/cache_utils.jl")
include("core/optimizer_stats.jl")
include("core/dataset.jl")
include("core/dataset_container.jl")

include("core/optimization_container.jl")
include("core/store_common.jl")

# Order Required
include("initial_conditions/initial_condition_chronologies.jl")

include("operation/problem_template.jl")
include("operation/operation_model_interface.jl")
include("operation/model_store_params.jl")
include("operation/abstract_model_store.jl")
include("operation/decision_model_store.jl")
include("operation/emulation_model_store.jl")
include("operation/initial_conditions_update_in_memory_store.jl")
include("operation/model_internal.jl")
include("operation/decision_model.jl")
include("operation/emulation_model.jl")
include("operation/problem_results_export.jl")
include("operation/problem_results.jl")
include("operation/operation_model_serialization.jl")
include("operation/time_series_interface.jl")
include("operation/optimization_debugging.jl")
include("operation/model_numerical_analysis_utils.jl")

include("initial_conditions/add_initial_condition.jl")
include("initial_conditions/update_initial_conditions.jl")
include("initial_conditions/calculate_initial_condition.jl")

include("parameters/add_parameters.jl")
include("parameters/update_parameters.jl")

include("feedforward/feedforwards.jl")
include("feedforward/feedforward_arguments.jl")
include("feedforward/feedforward_constraints.jl")

include("simulation/optimization_output_cache.jl")
include("simulation/optimization_output_caches.jl")
include("simulation/simulation_models.jl")
include("simulation/simulation_state.jl")
include("simulation/initial_condition_update_simulation.jl")
include("simulation/simulation_store_params.jl")
include("simulation/hdf_simulation_store.jl")
include("simulation/in_memory_simulation_store.jl")
include("simulation/simulation_problem_results.jl")
include("simulation/realized_meta.jl")
include("simulation/decision_model_simulation_results.jl")
include("simulation/emulation_model_simulation_results.jl")
include("simulation/simulation_sequence.jl")
include("simulation/simulation_internal.jl")
include("simulation/simulation.jl")
include("simulation/simulation_results_export.jl")
include("simulation/simulation_results.jl")

include("devices_models/devices/common/objective_functions.jl")
include("devices_models/devices/common/range_constraint.jl")
include("devices_models/devices/common/add_variable.jl")
include("devices_models/devices/common/add_auxiliary_variable.jl")
include("devices_models/devices/common/add_constraint_dual.jl")
include("devices_models/devices/common/rateofchange_constraints.jl")
include("devices_models/devices/common/duration_constraints.jl")
include("devices_models/devices/common/get_time_series.jl")

# Device Modeling components
include("devices_models/devices/interfaces.jl")
include("devices_models/devices/common/add_to_expression.jl")
include("devices_models/devices/common/set_expression.jl")
include("devices_models/devices/renewable_generation.jl")
include("devices_models/devices/thermal_generation.jl")
include("devices_models/devices/electric_loads.jl")
include("devices_models/devices/AC_branches.jl")
include("devices_models/devices/DC_branches.jl")
include("devices_models/devices/storage.jl")
include("devices_models/devices/hydro_generation.jl")
include("devices_models/devices/regulation_device.jl")
include("devices_models/devices/hybrid_generation.jl")

# Services Models
include("services_models/agc.jl")
include("services_models/reserves.jl")
include("services_models/group_reserve.jl")
include("services_models/service_slacks.jl")
include("services_models/services_constructor.jl")

# Network models
include("network_models/copperplate_model.jl")
include("network_models/powermodels_interface.jl")
include("network_models/pm_translator.jl")
include("network_models/network_slack_variables.jl")
include("network_models/area_balance_model.jl")

include("initial_conditions/initialization.jl")

# Device constructors
include("devices_models/device_constructors/constructor_validations.jl")
include("devices_models/device_constructors/thermalgeneration_constructor.jl")
include("devices_models/device_constructors/hydrogeneration_constructor.jl")
include("devices_models/device_constructors/branch_constructor.jl")
include("devices_models/device_constructors/renewablegeneration_constructor.jl")
include("devices_models/device_constructors/load_constructor.jl")
include("devices_models/device_constructors/storage_constructor.jl")
include("devices_models/device_constructors/regulationdevice_constructor.jl")
include("devices_models/device_constructors/hybridgeneration_constructor.jl")
# Network constructors
include("network_models/network_constructor.jl")

# Templates for Operation Problems
include("operation/operation_problem_templates.jl")

# Operations Decision Problems
include("operation/decision_problems.jl")

# Utils
include("utils/printing.jl")
include("utils/file_utils.jl")
include("utils/logging.jl")
include("utils/dataframes_utils.jl")
include("utils/jump_utils.jl")
include("utils/powersystems_utils.jl")
include("utils/recorder_events.jl")
include("utils/datetime_utils.jl")

end


R"""
rm(list=ls())
Start.Time <- Sys.time()

#load libraries
list.of.packages <- c("data.table","bit64","optimx","lme4",'languageR','lmerTest','glmnet','lsmeans','car','RDS','DMwR','stringr','nloptr','ggplot2','reshape2','plyr','viridis','gmodels')
new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
if(length(new.packages)) install.packages(new.packages)
lapply(list.of.packages,require,character.only=TRUE)

#######################################
#------------- Load Data -------------#
#######################################

# set working directory
setwd("/media/u01/analytics/scoring/CDW5_792")

# set path of data and headers location
input <- c("/media/u01/analytics/scoring/CDW5_792/")

# set output file path
output <- c("/media/u01/analytics/scoring/CDW5_792/Output/")

# set name of initial data file
dataname <- "csv_final_cdw5_792"

# set name of headers file
headers_data <- "Headers_cdw5.csv"

# read initial data and attach headers
finaldata <- fread(paste0(input,dataname),colClasses = "as.character",header=F)
datanames <- fread(paste0(input,headers_data),header=F)
setnames(finaldata,c(datanames$V1))

#check if response and mandatory variables exist in the data
#Put also the sub campaigns as mandatory
all_mandatory_vars <- c("Buyer_Pos_P1","Buyer_Pre_P1","Trps_POS_P1","Trps_PRE_P1","Dol_per_Trip_POS_P1","Dol_per_Trip_PRE_P1","Nonbuyer_Pre_P1","hh_age","estimated_hh_income","number_of_children_in_living_Un","person_1_gender","experian_id")
print(all_mandatory_vars %in% names(finaldata))

# set exposed flag variable
exposed_flag_var <- "exposed_flag_new"

# Put the sub campaigns and the demographic variables that will be used as random effects in the models
random_demos <- c("estimated_hh_income","hh_age","number_of_children_in_living_Un","person_1_gender")
random_campaigns <- c("Publisher_Fct1","Targeting_Fct1")

# set extra exposed flag variable to be excluded (in case you have more than one in dataset) and publisher variables that are not used in the mixed models
dropvars <- c("exposed_flag")

# set True if P2 category products are competitor or False otherwise (need for sign check in the model)
P2_Competitor <- TRUE #or FALSE

# pvalue threshold
pvalue_lvl <- 0.20

# Create folders for outputs
dir.create(paste0(output,"Descriptive Statistics for Publishers and Demographics"))
dir.create(paste0(output,"Final_Modelling_Results"))
dir.create(paste0(output,"Initial_Modelling_Results"))
dir.create(paste0(output,"QCs"))
dir.create(paste0(output,"saved_data"))
dir.create(paste0(output,"Scoring"))
final_modelling_output <- paste0(output,"Final_Modelling_Results/")
initial_modelling_output <- paste0(output,"Initial_Modelling_Results/")
scoring_output <- paste0(output,"Scoring/")
dir.create(paste0(scoring_output,"Occasion/"))
dir.create(paste0(scoring_output,"DollarsOccasion/"))
dir.create(paste0(scoring_output,"Penetration/"))
dir.create(paste0(final_modelling_output,"Occasion/"))
dir.create(paste0(final_modelling_output,"DollarsOccasion/"))
dir.create(paste0(final_modelling_output,"Penetration/"))
dir.create(paste0(initial_modelling_output,"Occasion/"))
dir.create(paste0(initial_modelling_output,"DollarsOccasion/"))
dir.create(paste0(initial_modelling_output,"Penetration/"))

scoring_Occ_output <- paste0(scoring_output,"Occasion/")
scoring_DolOcc_output <- paste0(scoring_output,"DollarsOccasion/")
scoring_Pen_output <- paste0(scoring_output,"Penetration/")
final_Occ_modelling_output <- paste0(final_modelling_output,"Occasion/")
final_DollOcc_modelling_output <- paste0(final_modelling_output,"DollarsOccasion/")
final_Pen_modelling_output <- paste0(final_modelling_output,"Penetration/")
initial_Occ_modelling_output <- paste0(initial_modelling_output,"Occasion/")
initial_DollOcc_modelling_output <- paste0(initial_modelling_output,"DollarsOccasion/")
initial_Pen_modelling_output <- paste0(initial_modelling_output,"Penetration/")

#######################################
#--------- Data Manipulation ---------#
#######################################

# create group, panid columns
finaldata[,`:=`(group=eval(parse(text=exposed_flag_var)),panid=experian_id)]

# remove not needed variables
if(length(dropvars)>0){
  for(i in noquote(dropvars)){
    finaldata <- finaldata[,noquote(i):=NULL]
  }
}

# mandatory variables for scoring
scoring_vars <- c("Prd_1_Net_Pr_PRE","Prd_1_Net_Pr_POS","Buyer_Pos_P0","Buyer_Pre_P0")

# Set the ProScore variable in the dataset
ProScore <- grep("MODEL",colnames(finaldata),value = TRUE)

# aggregate # of children for 4+
finaldata[number_of_children_in_living_Un>=4,number_of_children_in_living_Un:="4+" ]

# create factor variables
allrandoms <- c(random_demos,random_campaigns)
finaldata  <- as.data.frame(finaldata)
for(i in noquote(allrandoms)){
  finaldata[i] <- lapply(finaldata[i],function(x) as.factor(x))
}
finaldata  <- as.data.table(finaldata)

#     for(i in random_campaigns)
#     {
#       finaldata <- as.data.frame(finaldata)
#       finaldata[,i] <- as.factor(finaldata[,i])
#
#       # re-group random intercept levels
#       finaldata[,i] <- str_sub(finaldata[,i], start=1, end=2)
#       finaldata[,i] <- gsub("_","",finaldata[,i])
#       finaldata[,i] <- as.factor(finaldata[,i])
#
#       # create a csv file with the counts of each level of the publisher variable
#       #write.csv(table(finaldata$group,finaldata[,eval(parse(text=random_campaigns[i]))]),file="ExposureFreq5w.csv")
#
#       levelstokeep <- c("14","15","41","3","8","2","6","10","24")
#       all_levels <- unique(finaldata[,i])
#
#       other_levels <- setdiff(all_levels,c(levelstokeep,"\\N"))
#
#       finaldata <- as.data.table(finaldata)
#       for(j in other_levels){
#         finaldata <- finaldata[i==j,i:="Other"]
#       }
#
#     }

# GMT Check

# set variables as.numeric and replace NA's with zero
finaldata <- as.data.frame(finaldata)
indx <- sapply(finaldata, is.character)
options(warn=-1)
finaldata[indx] <- lapply(finaldata[indx],function(x) as.numeric(as.character(x)))
finaldata[,exposed_flag_var] <- NULL
options(warn=0)

# replace NAs with zero for numeric variables
finaldata <- as.data.frame(finaldata)
finaldata[sapply(finaldata, is.numeric)][is.na(finaldata[sapply(finaldata, is.numeric)])] <- 0
finaldata <- as.data.table(finaldata)

# set as zero the //N value of the group variable
finaldata <- finaldata[is.na(group),group:="0"]
finaldata <- finaldata[group=="\\N",group:="0"]
finaldata$group <- factor(finaldata$group)
finaldata$group <- relevel(finaldata$group,"0")

########## QCs ##########

# Create a summary of the dataset
t1 <- as.data.frame(finaldata[,sapply(finaldata,is.numeric)])
finaldata_num <- as.data.frame(finaldata)
finaldata_num <- finaldata_num[,c(which(t1==T))]
finaldata_num <- as.data.table(finaldata_num)
finaldata_num <- finaldata_num[,c(3:(which(names(finaldata_num)=="state")-1),((which(names(finaldata_num)=="mail_responder"))+1):length(names(finaldata_num))),with=F]
finaldata_num <- as.data.frame(finaldata_num)

summary_num <- sapply(finaldata_num[,!names(finaldata_num) %in% c("exposed_flag","experian_id","panid")],function(x) c(min(x,na.rm=T),quantile(x,c(0.25),na.rm=T),mean(x,na.rm=T),median(x,na.rm=T), quantile(x,c(0.75),na.rm=T),max(x,na.rm=T),sum(x,na.rm=T)))
rownames(summary_num) <- c("min","Q1","mean","median","Q3","max","sum")

na_count <-sapply(finaldata_num[,!names(finaldata_num) %in% c("exposed_flag","experian_id","panid")], function(y) sum(length(which(is.na(y)))))
NAs  <- as.vector(na_count)
N <- nrow(finaldata)-na_count

#write csv
write.csv(t(rbind(N,NAs,summary_num)),paste0(output,"QCs/","Summary.csv"),na='')

# Create Crosstables for campaign and pre campaign period
sink(paste0(output,"QCs/","PreCampaign_Summary.txt"))
CrossTable(finaldata$group,finaldata$Buyer_Pre_P1, expected = FALSE)
sink()
sink(paste0(output,"QCs/","Campaign_Summary.txt"))
CrossTable(finaldata$group,finaldata$Buyer_Pos_P1,expected = FALSE)
sink()

# Create check list for factors and Buyers
buy.pre <-  unique(finaldata$Buyer_Pre_P1)
buy.pos <- unique(finaldata$Buyer_Pos_P1)
non.pre <- unique(finaldata$Nonbuyer_Pre_P1)
gr <- unique(finaldata$group)
mos <-  unique(finaldata$Mosaic)
ban <- unique(finaldata$banner)
exp.uniq <- length(unique(finaldata$experian_id))

# check freq for demographic variables
sink(paste0(output,"QCs/","Demographics_Summary.txt"))
xtabs(~estimated_hh_income,finaldata)
cat("\n")
xtabs(~person_1_gender,finaldata)
cat("\n")
xtabs(~number_of_children_in_living_Un,finaldata)
cat("\n")
xtabs(~hh_age,finaldata)

cat("\n")
ftable(xtabs(~group+estimated_hh_income,finaldata))
cat("\n")
ftable(xtabs(~group+person_1_gender,finaldata))
cat("\n")
ftable(xtabs(~group+number_of_children_in_living_Un,finaldata))
cat("\n")
ftable(xtabs(~group+hh_age,finaldata))
sink()

finaldata_buyers <- finaldata[Buyer_Pos_P1=="1",]
sink(paste0(output,"QCs/","Demographics_Summary_Buyers_Campaign_Period.txt"))
cat("\n")
xtabs(~estimated_hh_income,finaldata_buyers)
cat("\n")
xtabs(~person_1_gender,finaldata_buyers)
cat("\n")
xtabs(~number_of_children_in_living_Un,finaldata_buyers)
cat("\n")
xtabs(~hh_age,finaldata_buyers)

cat("\n")
ftable(xtabs(~group+estimated_hh_income,finaldata_buyers))
cat("\n")
ftable(xtabs(~group+person_1_gender,finaldata_buyers))
cat("\n")
ftable(xtabs(~group+number_of_children_in_living_Un,finaldata_buyers))
cat("\n")
ftable(xtabs(~group+hh_age,finaldata_buyers))
sink()

sink(paste0(output,"QCs/","Buyers Data Metrics.txt"))
list(
  Mean_Dollars_Spent_per_Trip_POST_P1_Non_Exposed=mean(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  Mean_Dollars_Spent_per_Trip_PRE_P1_Non_Exposed=mean(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  Mean_Dollars_Spent_per_Trip_POST_P1_Exposed=mean(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  Mean_Dollars_Spent_per_Trip_PRE_P1_Exposed=mean(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  Mean_Trips_POST_P1_Non_Exposed=mean(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  Mean_Trips_PRE_P1_Non_Exposed=mean(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1),
  Mean_Trips_POST_P1_Exposed=mean(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  Mean_Trips_PRE_P1_Exposed=mean(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1),

  SD_Dol_per_Trip_POST_P1_Non_Exposed=sd(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  SD_Dol_per_Trip_PRE_P1_Non_Exposed=sd(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  SD_Dol_per_Trip_POST_P1_Exposed=sd(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  SD_Dol_per_Trip_PRE_P1_Exposed=sd(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  SD_Trips_POST_P1_Non_Exposed=sd(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  SD_Trips_PRE_P1_Non_Exposed=sd(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1),
  SD_Trips_POST_P1_Exposed=sd(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  SD_Trips_PRE_P1_Exposed=sd(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1),

  Min_Dol_per_Trip_POST_P1_Non_Exposed=min(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  Min_Dol_per_Trip_PRE_P1_Non_Exposed=min(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  Min_Dol_per_Trip_POST_P1_Exposed=min(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  Min_Dol_per_Trip_PRE_P1_Exposed=min(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  Min_Trips_POST_P1_Non_Exposed=min(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  Min_Trips_PRE_P1_Non_Exposed=min(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1),
  Min_Trips_POST_P1_Exposed=min(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  Min_Trips_PRE_P1_Exposed=min(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1),

  Max_Dol_per_Trip_POST_P1_Non_Exposed=max(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  Max_Dol_per_Trip_PRE_P1_Non_Exposed=max(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  Max_Dol_per_Trip_POST_P1_Exposed=max(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_POS_P1),
  Max_Dol_per_Trip_PRE_P1_Exposed=max(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Dol_per_Trip_PRE_P1),
  Max_Trips_POST_P1_Non_Exposed=max(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  Max_Trips_PRE_P1_Non_Exposed=max(finaldata[group=="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1),
  Max_Trips_POST_P1_Exposed=max(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_POS_P1),
  Max_Trips_PRE_P1_Exposed=max(finaldata[group!="0" & Buyer_Pos_P1=="1",]$Trps_PRE_P1)
)
sink()

cat("Please check summary.csv and proceed with the modeling process")

# remove HHs with no gender info
finaldata <- finaldata[!person_1_gender=="U",]

# aggregate U and L levels of hh income
finaldata <- finaldata[estimated_hh_income=="U",estimated_hh_income:="L"]

# check and drop exposed HHs with no publisher info or non-exposed HHs with publisher info
panid_out_all <- c()
if(length(random_campaigns)>0){
  for(k in random_campaigns){
    finaldata <- finaldata[eval(parse(text=k))=="\\N"|eval(parse(text=k))=="NULL"|eval(parse(text=k))=="0"|eval(parse(text=k))=="NONE",eval(parse(text=k)):="none"]
    cat(paste0(nrow(finaldata[(group=="0" & eval(parse(text=k))!="none")])," non exposed HHs with publisher info "))
    panid_out <- finaldata[(group=="0" & eval(parse(text=k))!="none"),panid]
    finaldata <- finaldata[!(group=="0" & eval(parse(text=k))!="none"),]
    cat('\n')
    # check and remove HHs that were exposed and we had no publisher info
    cat(paste0(nrow(finaldata[(group!="0" & eval(parse(text=k))=="none")])," exposed HHs with no publisher info "))
    finaldata <- finaldata[!(group!="0" & eval(parse(text=k))=="none")]
    panid_out2 <- finaldata[(group!="0" & eval(parse(text=k))=="none"),panid]
    panid_out_all <- unique(c(panid_out_all,panid_out,panid_out2))
  }
}

# apply matching rules for control group vs exposed (default is TRUE)
Match_control_exposed_groups <- TRUE  #FALSE or TRUE

if(Match_control_exposed_groups==T & length(ProScore)==0){
  finaldata <- as.data.table(finaldata)
  data_exposed <- finaldata[group!="0" ,]
  data_nonexposed1 <- finaldata[group=="0" & Buyer_Pre_P1=="1",]
  data_nonexposed0 <- finaldata[group=="0" & Buyer_Pre_P1=="0",]

  if(nrow(finaldata[group=="0",])>2000000){
    new_data_dim <- 0.3*nrow(finaldata[group=="0",])
  } else {
    if(nrow(finaldata[group=="0",])>1000000){
      new_data_dim <- 0.4*nrow(finaldata[group=="0",])
    } else {
      if(nrow(finaldata[group=="0",])>750000){
        new_data_dim <- 0.6*nrow(finaldata[group=="0",])
      } else {
        new_data_dim <- 0.7*nrow(finaldata[group=="0",])
      }
    }
  }

  dim_sample0 <- round(((new_data_dim-nrow(data_exposed))/(1+(nrow(data_exposed[Buyer_Pre_P1=="1",])/(nrow(data_exposed[Buyer_Pre_P1=="0",]))))))
  dim_sample1 <- new_data_dim-nrow(data_exposed)-dim_sample0

  new_data_nonexposed_11 <- as.data.frame(data_nonexposed1[sample(nrow(data_nonexposed1), dim_sample1),])
  new_data_nonexposed_01 <- as.data.frame(data_nonexposed0[sample(nrow(data_nonexposed0), dim_sample0),])
  data_exposed <- as.data.frame(data_exposed)
  finaldata_sample <- rbind(data_exposed,new_data_nonexposed_11,new_data_nonexposed_01)

  finaldata <- as.data.table(finaldata_sample)
}

if(Match_control_exposed_groups==T & length(ProScore)>0){
  exposed_proportion <- prop.table(xtabs(~eval(parse(text=ProScore)),finaldata[group!="0",]))
  proscore_levels <- names(exposed_proportion)

  if(nrow(finaldata[group=="0",])>2000000){
    new_data_dim <- 0.3*nrow(finaldata[group=="0",])
  } else {
    if(nrow(finaldata[group=="0",])>1000000){
      new_data_dim <- 0.4*nrow(finaldata[group=="0",])
    } else {
      if(nrow(finaldata[group=="0",])>800000){
        new_data_dim <- 0.6*nrow(finaldata[group=="0",])
      } else {
        new_data_dim <- 0.7*nrow(finaldata[group=="0",])
      }
    }
  }

  sample_control_data <- c()

  for (i in proscore_levels){
    sample_i_dim <- round(new_data_dim*exposed_proportion[names(exposed_proportion)==i])
    temp_data <- finaldata[eval(parse(text=ProScore))==i & group=="0",]
    data_i <- temp_data[sample(nrow(temp_data), sample_i_dim),]
    sample_control_data <- rbind(sample_control_data,data_i)
    #    cat(i)
    #    cat("\n")
    #    cat(length(sample_control_data$panid))
    #    cat("\n")
  }

  sample_data <- rbind(sample_control_data,finaldata[group!="0",])

  sample_data <- as.data.table(sample_data)
  data_exposed <- as.data.table(sample_data[group!="0",])

  data_nonexposed1 <- sample_data[group=="0" & Buyer_Pre_P1=="1",]
  data_nonexposed0 <- sample_data[group=="0" & Buyer_Pre_P1=="0",]

  if(nrow(data_nonexposed1)/nrow(data_nonexposed0) > nrow(data_exposed[Buyer_Pre_P1=="1",])/nrow(data_exposed[Buyer_Pre_P1=="0",])){
    new_data_nonexposed_11 <- as.data.frame(data_nonexposed1[sample(nrow(data_nonexposed1), (nrow(data_exposed[Buyer_Pre_P1=="1",])/nrow(data_exposed[Buyer_Pre_P1=="0",]))*nrow(data_nonexposed0)),])
    data_exposed <- as.data.frame(data_exposed)
    finaldata_sample <- rbind(data_exposed,new_data_nonexposed_11,data_nonexposed0)
  } else {
    new_data_nonexposed_01 <- as.data.frame(data_nonexposed0[sample(nrow(data_nonexposed0), (nrow(data_exposed[Buyer_Pre_P1=="0",])/nrow(data_exposed[Buyer_Pre_P1=="1",]))*nrow(data_nonexposed1)),])
    data_exposed <- as.data.frame(data_exposed)
    finaldata_sample <- rbind(data_exposed,new_data_nonexposed_01,data_nonexposed1)
  }
  length(finaldata_sample$panid)

  finaldata <- as.data.table(finaldata_sample)
}

# check proportions of the sample dataset
if(Match_control_exposed_groups==T ){
  finaldata_sample <- as.data.table(finaldata_sample)
  sink(paste0(output,"Matching_data_summary.txt"))
  if(length(ProScore)>0){
    cat("\n")
    xt <- xtabs(~eval(parse(text=ProScore)),finaldata_sample[group!="0",])
    names(dimnames(xt)) <- c("Proportions of ProScore in the exposed group")
    print(prop.table(xt))
    cat("\n\n")
    xt <- xtabs(~eval(parse(text=ProScore)),finaldata_sample[group=="0",])
    names(dimnames(xt)) <- c("Proportions of ProScore in the control group")
    print(prop.table(xt))
  }
  cat("\n\n")
  xt <- xtabs(~Buyer_Pre_P1,finaldata_sample[group!="0",])
  names(dimnames(xt)) <- c("Percentages of target product buyers during the Pre-Campaign Period in the exposed group")
  print(prop.table(xt))
  cat("\n\n")
  xt <- xtabs(~Buyer_Pre_P1,finaldata_sample[group=="0",])
  names(dimnames(xt)) <- c("Percentages of target product buyers during the Pre-Campaign Period in the control group")
  print(prop.table(xt))
  cat("\n\n")
  cat('Average_Trips_PRE_P1_Non_Exposed_matched_data')
  cat("\n")
  cat(mean(finaldata_sample[Buyer_Pre_P1=="1" & group=="0",]$Trps_PRE_P1))
  cat("\n\n")
  cat('Average_Trips_PRE_P1__Exposed_matched_data')
  cat("\n")
  cat(mean(finaldata_sample[Buyer_Pre_P1=="1" & group!="0",]$Trps_PRE_P1))
  cat("\n\n")
  cat('Average_Spent_PRE_P1_Non_Exposed_matched_data')
  cat("\n")
  cat(mean(finaldata_sample[Buyer_Pre_P1=="1" & group=="0",]$Dol_per_Trip_PRE_P1))
  cat("\n\n")
  cat('Average_Spent_PRE_P1_Exposed_matched_data')
  cat("\n")
  cat(mean(finaldata_sample[Buyer_Pre_P1=="1" & group!="0",]$Dol_per_Trip_PRE_P1))
  sink()
}

# remove demographic variables
finaldata <- droplevels(finaldata)
finaldata$panid <- as.character(finaldata$panid)
initial_data <- finaldata[,c(4:(which(names(finaldata)=="state")-1),which(names(finaldata)=="person_1_gender"),which(names(finaldata)=="number_of_children_in_living_Un"),(which(names(finaldata)=="Mosaic")+1):length(names(finaldata))),with=F]
names(initial_data)
options(warn=-1)

#GMT
xVarsDemos <- setdiff(names(finaldata),names(initial_data))

######rm(finaldata,data_nonexposed1,temp_data,data_i,sample_control_data,sample_data,data_exposed,data_nonexposed0,finaldata_sample,finaldata_num,new_data_nonexposed_11,new_data_nonexposed_01)
options(warn=0)

# get number of products in the data
num_products <- length(grep("Buyer_Pre_P",colnames(initial_data)))-1

#exclude POST variables
indexPOS <- grep("POS",colnames(initial_data))
indexPos <- grep("Pos",colnames(initial_data))
indexPre1 <- grep("Buyer_Pre_",colnames(initial_data))
indexPre2 <- grep("Nonbuyer_Pre_",colnames(initial_data))
# check pre vars to exclude (only buyer_pre and nonbuyer_pre should be excluded)
(colnames(initial_data)[c(indexPre1,indexPre2)])
initial_vars_to_exclude <- colnames(initial_data)[c(indexPOS,indexPos,indexPre1,indexPre2)]
indexPREPOS <- grep("PRE_POS",colnames(initial_data))
indexPrePos <- grep("Pre_Pos",colnames(initial_data))
PREPOS_vars_to_include <- colnames(initial_data)[c(indexPREPOS,indexPrePos)]
ALL_vars_to_exclude <- setdiff(initial_vars_to_exclude,c(PREPOS_vars_to_include,all_mandatory_vars,scoring_vars))
initial_data <- as.data.frame(initial_data)
initial_data <- initial_data[ , !names(initial_data) %in% c(ALL_vars_to_exclude)]

#exclude category variables(exclude P0's)
indexP0 <- grep("0",colnames(initial_data))
P0_vars_to_exclude <- colnames(initial_data)[indexP0]
indexMODEL <- grep("MODEL",colnames(initial_data))
ProScore_vars_to_include <- colnames(initial_data)[indexMODEL]
P0_vars_to_exclude <- setdiff(P0_vars_to_exclude,c(ProScore_vars_to_include,all_mandatory_vars,scoring_vars))
initial_data <- initial_data[ , !names(initial_data) %in% c(P0_vars_to_exclude)]

#exclude new variables that are only for reporting
indexPERC <- grep("Perc_",colnames(initial_data))
indexPriPerUn <- grep("Pr_per_",colnames(initial_data))
vars_to_exclude <- colnames(initial_data)[c(indexPERC,indexPriPerUn)]
initial_data <- initial_data[ , !names(initial_data) %in% c(vars_to_exclude)]

# segments for outliers detection
initial_data <- as.data.table(initial_data)
data_NB_NE_B <- initial_data[Buyer_Pre_P1=="0" & group=="0"  & Buyer_Pos_P1=="1",]
#data_B_E_NB <- initial_data[Buyer_Pre_P1=="1" & group!="0" & Buyer_Pos_P1=="0",]
pen_reduction <- initial_data[Buyer_Pre_P1=="1" & group=="0" & Buyer_Pos_P1=="0",]
occ_reduction <- initial_data[group=="0" & Buyer_Pos_P1=="1" & Trps_POS_P1<Trps_PRE_P1,]
dol_reduction <- initial_data[group=="0" & Buyer_Pos_P1=="1" & Dol_per_Trip_POS_P1<Dol_per_Trip_PRE_P1,]

# get variables that need to have negative sign
negative_index <- c()
for(i in 3:num_products){
  indexCompetitorVars <- grep(paste0("P",i),colnames(initial_data))
  indexCompetitorVars2 <- grep(paste0(i,"_"),colnames(initial_data))
  indexGAS <- grep("AVG_PRICE",colnames(initial_data))
  negative_index <- c(negative_index,indexCompetitorVars,indexCompetitorVars2,indexGAS)
}
negativevars <- unique(colnames(initial_data)[negative_index])

if(P2_Competitor==T){
  indexNOCompetitorVars1 <- grep("P2",colnames(initial_data))
  indexNOCompetitorVars2 <- grep(paste0(2,"_"),colnames(initial_data))
  indexNOCompetitorVars <- c(indexNOCompetitorVars1,indexNOCompetitorVars2)
  NOCompetitorVars <- colnames(initial_data)[indexNOCompetitorVars]
  negativevars <- unique(c(negativevars,NOCompetitorVars))
}

# get variables that need to have positive sign
positive_index1 <- grep("P1",colnames(initial_data))
positive_index2 <- grep("1_",colnames(initial_data))
positive_index3 <- grep("MODEL",colnames(initial_data))
positive_index <- c(positive_index1,positive_index2,positive_index3)
positivevars <- colnames(initial_data)[positive_index]

# # kruskal wallis test for pair comparison of publisher's levels keep aggregated levels based on the spent aggregation
# kruskal_var <- 'Dol_per_Trip_POS_P1'
# buyers_data <- initial_data[Buyer_Pos_P1=='1',]
# kruskal_randoms <- c()
# for(i in random_campaigns){
#   kt <- kruskal.test(eval(parse(text=kruskal_var))~eval(parse(text=i)),data=buyers_data)
#   cat(i);print(kt)
#   if(kt$p.value<0.10){
#     kruskal_randoms <- append(kruskal_randoms,i)
#   } else {
#     initial_data[,eval(parse(text=i)):=NULL]
#   }
# }
#
# warning('In case of aggregation based on Kruskal-Wallis test, make sure that the other level of each sub-campaign is named as Other_XXX')
# for(i in kruskal_randoms){
#   lev <- setdiff(levels(buyers_data[,eval(parse(text=i))]),'none')
#   if(length(lev)>2){
#     aggr <- c()
#     for(j in 1:(length(lev)-1)){
#       for( k in (j+1):(length(lev))){
#         df <- subset(buyers_data, eval(parse(text=i))  %in% c(lev[j],lev[k]))
#         l1 <- list(c(lev[j],lev[k]),P_Value=try(round(kruskal.test(df[,eval(parse(text=kruskal_var))]~df[,eval(parse(text=i))])$p.value,5),silent=TRUE))
#         if(l1[2]=="NaN") next
#         vectorElements <- unlist(l1)
#         listnames <- names(vectorElements) <- c(paste0(i,':'),paste0(i,':'),'P_value:')
#         listout <- paste(listnames,vectorElements)
#         write.table(listout,file=paste0(output,'Kruskal.csv'), row.names = FALSE, col.names = FALSE, quote = FALSE,append=TRUE)
#         if(l1[['P_Value']]<0.05){
#           aggr <- append(aggr,c(sapply(l1[1],`[`,1),sapply(l1[1],`[`,2)))
#           aggr <- setdiff(aggr,c('none',paste0("Other_",i)))
#         }
#       }
#     }
#     if(length(unique(aggr))>0){
#       initial_data[!eval(parse(text=i)) %in% c(aggr,'none'),eval(parse(text=i)):=paste0("Other_",i)]
#     } else {
#       initial_data[,eval(parse(text=i)):=NULL]
#     }
#   }
# }

initial_data <- droplevels(initial_data)
saveRDS(initial_data,"initial_data.RDS")
random_campaigns <- intersect(random_campaigns,names(initial_data))

if(length(random_campaigns)>0){
  random_campaigns_factors <- paste0("+(0+group|",random_campaigns,")")
} else {
  random_campaigns_factors <- NULL
}
random_demos_factors <- paste0( "+(0+group|",random_demos,")" )

# Create Descriptives for Campaign breaks
if(length(random_campaigns)>0){
  output_checks <- paste0(output,"Descriptive Statistics for Publishers and Demographics/")
  randoms <- c(random_demos,random_campaigns)
  for(i in randoms){
    write.csv(table(initial_data[,eval(parse(text=i))]),file=paste0(output_checks,i,'.csv'),row.names=F)
    write.csv(table(initial_data$group,initial_data[,eval(parse(text=i))]),file=paste0(output_checks,i,'_group.csv'))

    dolocc_data <- initial_data[Buyer_Pos_P1=="1",]

    write.csv(table(dolocc_data[,eval(parse(text=i))]),file=paste0(output_checks,i,'_buyers.csv'),row.names=F)
    write.csv(table(dolocc_data$group,dolocc_data[,eval(parse(text=i))]),file=paste0(output_checks,i,'_group_buyers.csv'))
  }

  # check exposed and non exposed HHs counts for target brand Buyers in the campaign period
  write.csv(table(initial_data$group,initial_data$Buyer_Pos_P1),file=paste0(output_checks,'buyers_pos_group.csv'))

  # check exposed and non exposed HHs counts for target brand Buyers in the pre-campaign period
  write.csv(table(initial_data$group,initial_data$Buyer_Pre_P1),file=paste0(output_checks,'buyers_pre_group.csv'))

  # calculate correlations between levels of the random intercept variables
  finaldata_cor <- as.data.frame(initial_data)

  for(i in randoms){
    for(level in unique(eval(parse(text=paste0("finaldata_cor$",i))))){
      finaldata_cor[paste(i, level, sep = "_")] <- ifelse(eval(parse(text=paste0("finaldata_cor$",i))) == level, 1, 0)
    }
  }

  cor.data <- finaldata_cor[,c((which(names(finaldata_cor)=="panid")+1):length(finaldata_cor))]
  cor.matrix <- cor(cor.data)
  write.csv(cor.matrix,file=paste0(output_checks,"random_correlation_matrix.csv"))

  mel.cc <- melt(cor.matrix)
  heat_map <- ggplot(mel.cc,aes(Var1, Var2))+geom_tile(aes(fill=value))+ scale_fill_viridis(name="Correlation",option="magma") + theme(axis.text.x = element_text(angle=90, vjust=0.5))+ coord_equal()
  ggsave(filename=paste0(output_checks,"correlation_heat_map.pdf"),heat_map,width = 10, height = 10, units = "in")

  #Make descriptive plots for publishers, creatives etc.
  # create descriptive plots for the DolOcc breaks
  buyers_data <- initial_data[Buyer_Pos_P1=="1",]
  for(i in random_campaigns){
    buyers_data <- buyers_data[eval(parse(text=i))!="none",]
    buyers_data <- droplevels(buyers_data)
    buyers_data <- as.data.frame(buyers_data)
    res <- ldply(levels(buyers_data[,i]),function(x) round(mean(buyers_data[buyers_data[,i]==x,]$Dol_per_Trip_POS_P1),2))
    df <-cbind(res,levels(buyers_data[,i]))
    colnames(df) <- c('Average_Spent_during_the_campaign_period','Levels')

    p <- ggplot(df,aes(x=Levels,y=Average_Spent_during_the_campaign_period))+geom_bar(stat='identity')+ theme(axis.text.x = element_text(angle=90, vjust=0.5))+ylab('Average Spent for the target brand during the campaign period')+
      theme(legend.position="none")+theme(panel.background = element_rect(fill = 'white'))+
      geom_text(stat='identity',aes(label=Average_Spent_during_the_campaign_period,group=Levels), position = position_dodge(0.9), vjust = 0)
    ggsave(filename=paste0(output_checks,i,'_Mean_Spent.pdf'),p,height = 8,width = 15)

    p <- ggplot(buyers_data,aes_string(i,'Dol_per_Trip_POS_P1'))+geom_boxplot()+theme(axis.text.x = element_text(angle = 90, vjust = 0.5))+
      ylab('Spent per Trip for the target bramd during the campaign period')
    ggsave(filename=paste0(output_checks,i,'_Spent_BoxPlot.pdf'),p,height = 8,width = 15)
    buyers_data <- as.data.table(buyers_data)
  }

  # create descriptive plots for the Occ breaks
  options(warn=-1)
  a1 <- buyers_data[Trps_POS_P1>=5,Trps_POS_P1:="5"]
  options(warn=0)
  a1[,Trips_growth:=log(Trps_POS_P1/(Trps_PRE_P1+1))]
  a1$Trps_POS_P1 <- as.factor(a1$Trps_POS_P1)
  a1 <- a1[Trps_POS_P1=="5",Trps_POS_P1:="5+"]

  response_var <- c("Trps_POS_P1")
  for(i in random_campaigns){
    a1 <- as.data.table(a1)
    a1 <- a1[eval(parse(text=i))!="none",]
    a1 <- droplevels(a1)
    rr <- c(response_var,i)
    a1 <- as.data.frame(a1)
    df <- a1[,rr]
    mel <- melt(df,id.var=i)
    p <- ggplot(mel,aes_string(i))+geom_bar(aes(y=..count..,fill=as.factor(value)),position=position_dodge(0.9))+
      theme(axis.text.x = element_text(angle = 90, vjust = 0.5))+guides(fill=guide_legend(title=response_var))+
      geom_text(stat='count',aes(label=..count..,group=value), position = position_dodge(0.9), vjust = 0)
    ggsave(filename=paste0(output_checks,i,'_Trips.pdf'),p,height = 8,width = 15)

    a2 <- as.data.table(initial_data[Buyer_Pos_P1=="1",])
    a2 <- a2[eval(parse(text=i))!="none",]
    a2 <- as.data.frame(a2)
    a2 <- droplevels(a2)
    res <- ldply(levels(a2[,i]),function(x) round(mean(a2[a2[,i]==x,]$Trps_POS_P1),2))
    df <-cbind(res,levels(a2[,i]))
    colnames(df) <- c('Average_Trips_during_the_campaign_period','Levels')
    p <- ggplot(df,aes(x=Levels,y=Average_Trips_during_the_campaign_period))+geom_bar(stat='identity')+ theme(axis.text.x = element_text(angle=90, vjust=0.5))+ylab('Average Trips for the target brand during the campaign period')+
      theme(legend.position="none")+theme(panel.background = element_rect(fill = 'white'))+
      geom_text(stat='identity',aes(label=Average_Trips_during_the_campaign_period,group=Levels), position = position_dodge(0.9), vjust = 0)
    ggsave(filename=paste0(output_checks,i,'_Mean_Trips.pdf'),p,height = 8,width = 15)
  }

  # create descriptive plots for the Pen breaks
  response_var <- c("Buyer_Pos_P1")
  for(i in random_campaigns){
    data <- as.data.table(copy(initial_data))
    data <- data[eval(parse(text=i))!="none",]
    data <- droplevels(data)
    data <- as.data.frame(data)
    rr <- c(response_var,i)
    df <- data[,rr]
    mel <- melt(df,id.var=i)
    p <- ggplot(mel,aes_string(x=i))+geom_bar(aes(y=..count..,fill=as.factor(value)),position=position_dodge(0.9))+
      theme(axis.text.x = element_text(angle = 90, vjust = 0.5))+guides(fill=guide_legend(title=response_var))+
      geom_text(stat='count',aes(label=..count..,group=value), position = position_dodge(0.9), vjust = 0)
    ggsave(paste0(output_checks,i,'_Buyers_during_Campaign.pdf'),p,height = 8,width = 15)
  }
}

rm(dolocc_data,mel.cc,finaldata_cor,cor.data,cor.matrix,buyers_data,a1,a2,data)

#Create functions for modelling process and outputs
ggCaterpillar <- function(re,  QQ=FALSE, likeDotplot=FALSE) {
  require(ggplot2)
  f <- function(x) {
    pv   <- attr(x, "postVar")
    cols <- 1:(dim(pv)[1])
    se   <- unlist(lapply(cols, function(i) sqrt(pv[i, i, ])))
    ord  <- unlist(lapply(x, order)) + rep((0:(ncol(x) - 1)) * nrow(x), each=nrow(x))
    pDf  <- data.frame(y=unlist(x)[ord],
                       ci=1.28*se[ord], #80%CI:1.28, 90%CI:1.645, 95%CI: 1.96
                       nQQ=rep(qnorm(ppoints(nrow(x))), ncol(x)),
                       ID=factor(rep(rownames(x), ncol(x))[ord], levels=rownames(x)[ord]),
                       ind=gl(ncol(x), nrow(x), labels=names(x)))

    if(QQ) {  ## normal QQ-plot
      p <- ggplot(pDf, aes(nQQ, y))
      p <- p + facet_wrap(~ ind, scales="free")
      p <- p + xlab("Standard normal quantiles") + ylab("Random effect quantiles")
    } else {  ## caterpillar dotplot
      p <- ggplot(pDf, aes(ID, y)) #+ coord_flip()
      if(likeDotplot) {  ## imitate dotplot() -> same scales for random effects
        p <- p + facet_wrap(~ ind)
      } else {           ## different scales for random effects
        p <- p + facet_wrap(~ ind, scales="free_y")
      }
      p <- p + xlab("Levels") + ylab("Random effects")
    }

    p <- p + theme(legend.position="none")
    p <- p + geom_hline(yintercept=0)
    p <- p + geom_errorbar(aes(ymin=y-ci, ymax=y+ci), width=0, colour="black")
    p <- p + geom_point(aes(size=1.2), colour="blue") + theme(axis.text.x = element_text(angle=90, vjust=0.5))

    return(p)
    #ggsave(paste0(output,"dotplots.pdf"),p)
  }

  lapply(re,f)
}

se.ranef <- function (object) {
  se.bygroup <- ranef(object, condVar = TRUE)
  n.groupings <- length(se.bygroup)
  for (m in 1:n.groupings) {
    vars.m <- attr(se.bygroup[[m]], "postVar")
    K <- dim(vars.m)[1]
    J <- dim(vars.m)[3]
    se.bygroup[[m]] <- array(NA, c(J, K))
    for (j in 1:J) {
      se.bygroup[[m]][j, ] <- sqrt(diag(as.matrix(vars.m[,, j])))  }
    names.full <- dimnames(se.bygroup)
    dimnames(se.bygroup[[m]]) <- list(names.full[[1]], names.full[[2]]) }
  return(se.bygroup) }

# create nloptwrap optimization method
defaultControl <- list(algorithm="NLOPT_LN_BOBYQA",xtol_rel=1e-6,maxeval=1e5)
nloptwrap2 <- function(fn,par,lower,upper,control=list(),...) {
  for (n in names(defaultControl))
    if (is.null(control[[n]])) control[[n]] <- defaultControl[[n]]
    res <- nloptr(x0=par,eval_f=fn,lb=lower,ub=upper,opts=control,...)
    with(res,list(par=solution,
                  fval=objective,
                  feval=iterations,
                  conv=if (status>0) 0 else status,
                  message=message))
}

augment.ranef.mer <- function(x,
                              ci.level=0.9,
                              reorder=TRUE,
                              order.var=1) {
  tmpf <- function(z) {
    if (is.character(order.var) && !order.var %in% names(z)) {
      order.var <- 1
      warning("order.var not found, resetting to 1")
    }
    ## would use plyr::name_rows, but want levels first
    zz <- data.frame(level=rownames(z),z,check.names=FALSE)
    if (reorder) {
      ## if numeric order var, add 1 to account for level column
      ov <- if (is.numeric(order.var)) order.var+1 else order.var
      zz$level <- reorder(zz$level, zz[,order.var+1], FUN=identity)
    }
    ## Q-Q values, for each column separately
    qq <- c(apply(z,2,function(y) {
      qnorm(ppoints(nrow(z)))[order(order(y))]
    }))
    rownames(zz) <- NULL
    pv   <- attr(z, "postVar")
    cols <- 1:(dim(pv)[1])
    se   <- unlist(lapply(cols, function(i) sqrt(pv[i, i, ])))
    ## n.b.: depends on explicit column-major ordering of se/melt
    zzz <- cbind(melt(zz,id.vars="level",value.name="estimate"),
                 qq=qq,std.error=se)
    ## reorder columns:
    subset(zzz,select=c(variable, level, estimate, qq, std.error))
  }
  dd <- ldply(x,tmpf,.id="grp")
  ci.val <- -qnorm((1-ci.level)/2)
  transform(dd,
            p=2*pnorm(-abs(estimate/std.error)), ## 2-tailed p-val
            lb=estimate-ci.val*std.error,
            ub=estimate+ci.val*std.error)
}

"""


# Based on
# https://gitlab.com/agentydragon/solid-flask/-/blob/master/solid_flask_main.py

# This implementation is not secure.
# Many required cryptographic checks were ignored.

using HTTP
using JSON
using Base64
using Random
using SHA
using Sockets
using PyCall
using Dates

const pyjwk = pyimport("jwcrypto.jwk")
const pyjwt = pyimport("jwcrypto.jwt")

const _ISSUER        = "https://inrupt.net/"
const _PORT          = 3333
const _REDIRECT_PATH = "/oauth/callback"
const _REDIRECT_URI  = "http://localhost:" * string(_PORT) * _REDIRECT_PATH
const _AUTHORITY     = _ISSUER * ".well-known/openid-configuration"

const random_device = RandomDevice()
const state_storage = Dict{String,Dict{String,String}}()
const access_token  = Ref("")
const key           = Ref("")

function register_client(provider_info)
    url     = provider_info["registration_endpoint"]
    headers = ["content-type" => "application/json"]
    data    = Dict("redirect_uris" => [_REDIRECT_URI])
    result  = JSON.parse(String(HTTP.post(url, headers, JSON.json(data)).body))
    @info "registration response" result
    return result["client_id"]
end

function make_random_string()
    randstring(random_device, 50)
end

function make_verifier_challenge()
    code_verifier  = make_random_string()
    code_challenge = base64encode(sha256(code_verifier))
    code_challenge = replace(replace(replace(code_challenge, "+" => "-"), "/" => "_"), "=" => "")
    code_verifier, code_challenge
end

const provider_info = JSON.parse(String(HTTP.request("GET", _AUTHORITY).body))
@info "provider info" provider_info

const client_id = register_client(provider_info)

function template(web_id, resource, resource_content)
    web_str = web_id != "" ?
        "You are logged in as $(web_id)." :
        "You are not logged in."
    res_str = resource != "" ?
        "
$(resource_content)
" :
        "Use the form below to read a resource."

    """

Login status

    $(web_str)

Resource content

    $(res_str)

    <form action=/ method=GET>
      <input
          value="$(resource)"
          placeholder='https://you.solidcommunity.net/private/...'
          name='resource'>
      <input type=submit value=Read>
    </form>
    """
end

function make_token_for(pykeypair, uri, method)
    header = Dict(
        "typ" => "dpop+jwt",
        "alg" => "ES256",
        "jwk" => pykeypair.export(private_key=false, as_dict=true)
    )
    claims = Dict(
        "jti" => make_random_string(),
        "htm" => method,
        "htu" => uri,
        "iat" => Int(floor(datetime2unix(now())))
    )
    jwt = pyjwt.JWT(header=header, claims=claims)
    jwt.make_signed_token(pykeypair)
    return jwt.serialize()
end

function index(req::HTTP.Request)
    queryparams = HTTP.queryparams(HTTP.URI(req.target))
    tested_url  = get(queryparams, "resource", "")

    if access_token[] != "" && key[] != ""
        println("loading access token and key from session")
        keypair = pyjwk.JWK.from_json(key[])
        headers = [
            "authorization" => "DPoP " * access_token[],
            "dpop"          => make_token_for(keypair, tested_url, "GET")
        ]
        decoded_access_token = pyjwt.JWT(jwt=access_token[])
        web_id = JSON.parse(decoded_access_token.token.objects["payload"])["sub"]
    else
        headers = Pair{String, String}[]
        web_id = ""
    end

    if tested_url != ""
        resp = HTTP.get(tested_url, headers; status_exception = false)
        if resp.status == 401
            println("Got 401 trying to access " * tested_url)
            code_verifier, code_challenge = make_verifier_challenge()
            state = make_random_string()
            @assert !haskey(state_storage,state)
            state_storage[state] = Dict("code_verifier" => code_verifier, "redirect_url" => req.target)

            url = provider_info["authorization_endpoint"] * "?" * HTTP.escapeuri([
                "code_challenge"        => code_challenge,
                "state"                 => state,
                "response_type"         => "code",
                "redirect_uri"          => _REDIRECT_URI,
                "code_challenge_method" => "S256",
                "client_id"             => client_id,
                "scope"                 => "openid offline_access"
            ])
            return HTTP.Response(302,["Location" => url])
        elseif resp.status != 200
            error("Unexpected status code: $(resp.status) $(resp.body)")
        end
        resource_content = String(resp.body)
    else
        resource_content = ""
    end
    headers = ["Content-Type" => "text/html"]
    return HTTP.Response(200, headers; body=template(web_id,tested_url,resource_content))
end

function oauth_callback(req::HTTP.Request)
    queryparams = HTTP.queryparams(HTTP.URI(req.target))
    auth_code   = get(queryparams, "code", "")
    state       = get(queryparams, "state", "")

    @assert haskey(state_storage, state) "state $(state) not in state_storage?"

    pykeypair     = pyjwk.JWK.generate(kty="EC", crv="P-256")
    key[]         = pykeypair.export()

    state_info    = state_storage[state]
    code_verifier = state_info["code_verifier"]
    redirect_url  = state_info["redirect_url"]

    url     = provider_info["token_endpoint"]
    headers = [
        "dpop" => make_token_for(pykeypair, url, "POST"),
        "content-type" => "application/x-www-form-urlencoded"
    ]
    data    = HTTP.escapeuri([
        "grant_type"    => "authorization_code",
        "client_id"     => client_id,
        "redirect_uri"  => _REDIRECT_URI,
        "code"          => auth_code,
        "code_verifier" => code_verifier
    ])
    result = JSON.parse(String(HTTP.post(url, headers, data; redirect=false).body))
    access_token[] = result["access_token"]

    pop!(state_storage, state)
    return HTTP.Response(302, ["Location" => redirect_url])
end

const ROUTER = HTTP.Router()
HTTP.@register(ROUTER, "GET", "/", index)
HTTP.@register(ROUTER, "GET", "/oauth/callback", oauth_callback)
@async HTTP.serve(ROUTER, ip"127.0.0.1", _PORT)


function main(args)
    return args
end
# Load inline arguments for worker unit
num_workers, this = main(ARGS)
num_workers = parse(Int64, num_workers)
this        = parse(Int64, this)

#
# Load packages and functions
include("../../src/init.jl")
include("../../src/fct.jl")
include("../../src/worker_fct.jl")
# using Distributed
#
# Output paths and filter settings
plots_base_path = "../../../waveforms/baselines/plots/raw/"
base_path_raw   = "../../../waveforms/baselines/raw_wf/"
base_path       = "../../../waveforms/baselines/wf/"
# addprocs(3)
# nprocs()

cal = JSON.parsefile("../../dicts/A_cal.json");
channels = 0:1:36
channels = get_share_for_worker(channels, num_workers, this)
for ch in channels
    files = glob(base_path_raw * "*/" * lpad(ch,2,"0") * "-" * channel_to_name[ch] * "/*.h5");
#     IJulia.clear_output(true)
    Base.run(`clear`)
    @info("Channel Ch" * lpad(ch, 2, "0") * " in progress")
    @showprogress for file in files
        filename = base_path * split(split(file, "/raw_wf/")[2], "-bl-raw.h5")[1] * ".h5"
        run_str = split(basename(file), "-")[2]
        run = parse(Int64, split(run_str, "run")[end])
        if haskey(cal, channel_to_name[ch])
            if haskey(cal[channel_to_name[ch]], run_str) && !isfile(filename)
                data = HDF5.h5open(file, "r") do h5f
                    LegendHDF5IO.readdata(h5f, "data")
                end
                cal_pulses = []
                map(x->push!(cal_pulses, RDWaveform(x.time, (float.(x.value) .- mean(x.value)) ./ cal[channel_to_name[ch]][run_str][1] )), data.waveform);

                !isdir(dirname(filename)) ? mkpath(dirname(filename)) : ""
                HDF5.h5open(filename, "w") do h5f
                    LegendHDF5IO.writedata(h5f, "data", Table(
                            energy = data.energy,
                            multiplicity = data.multiplicity,
                            timestamp = data.timestamp,
                            run = data.run,
                            datasetID = data.datasetID,
                            channel = data.channel,
                            waveform = StructArray{RDWaveform}(Array{RDWaveform,1}(cal_pulses))
                        )
                    )
                end
            end
        end
    end
end


#---- Set up
println("\n\n\n Analysis begins:")
cd("/Users/yanivabir/Files/ShohamyLab/DecisionToExplore/exp4_replication/decExp4")
# Criteria
deck_ratio_deviation = 0.3
reward = 0.25

# Load libraries
using CSV, DataFrames, Statistics, Distributions

#---- Get data
date = r"2021-11-04|2021-11-05"
sess = "4"
subdir = "now"

# List relevant files
files = filter(x -> occursin(date, x) &&
    occursin("sess" * sess, x) &&
    !occursin("plan", x) &&
    !occursin("int", x) &&
    occursin("csv", x), readdir("../Data/"* subdir* "/"))

# Open each file, and then vcat them together
dat = ((x -> DataFrame!(CSV.File("../Data/" * subdir * "/" * x))).(files)) |> d -> vcat(d..., cols = :union)

#---- Exclude Ps who didn't finish

quality = combine(groupby(dat, :PID), :n_warnings => maximum => :n_w) # Summarize warnings each participant recieved

kick = combine(groupby(dat, :PID), :category => (x -> "kick-out" in x) => :kick)
filter!(x -> !ismissing(x.kick) && x.kick == true, kick)

filter!(r -> !(r.PID in kick.PID), dat) # Filter data according to quality table

println("\nExcluded during experiment:")
println(kick.PID)
#---- Prepare data
# Remove instructions and training
a = filter(r -> r.trial_type == "structure-quiz", dat) |>  # Find structure-quiz as beginning of interesting data
    x -> combine(groupby(x, [:PID, :sess]), :trial_index => minimum => :start_trial) # Find fist appearance of
dat = innerjoin(dat, a, on = [:PID, :sess]) |> # Join with data
    x -> filter!(r -> r.trial_index >= r.start_trial, x) # filter data

# Deal with missing block number data
for ii in 1:nrow(dat)
    if ismissing(dat[ii, :block])
        dat[ii, :block] = dat[ii - 1, :block]
    end
end

# Make deck names into numbers
dat.LCard = passmissing(x -> parse(Int64, x[8:end])).(dat.LCard)
dat.RCard = passmissing(x -> parse(Int64, x[8:end])).(dat.RCard)
dat.chosen_deck = passmissing(x -> parse(Int64, x[8:end])).(dat.chosen_deck)

# Make table names into numbers
dat.TL = passmissing(x -> parse(Int64, x[end])).(dat.TL)
dat.TR = passmissing(x -> parse(Int64, x[end])).(dat.TR)
dat.chosen_object = passmissing(x -> parse(Int64, x[end])).(dat.chosen_object)
dat.table = passmissing(x -> parse(Int64, x[end])).(dat.table)

# Which deck was chosen?
dat.chosen_deck =
    (x -> !ismissing(x.chosen_deck) ? x.chosen_deck : (ismissing(x.chosen_deck_side) ? missing :
        (x.chosen_deck_side == "L" ? x.LCard : x.RCard))).(eachrow(dat))

#---- Compute bonus payments
bonus = filter(x -> x.trial_type == "test-feedback", dat) |>
    x -> combine(groupby(x, :PID), :correct => (x -> sum((x .== "true") .| (x .== true))) => :correct)

bonus.reward = (x -> round(x, digits = 2)).(bonus.correct .* reward)
if sess == "4"
    bonus.reward = bonus.reward .+ 2
end

println("\nBonuses:")
println(sort(bonus, [:reward]))
if subdir != "raw"
    CSV.write("../Data/" * subdir * "/bonus.csv", sort(bonus, [:reward])[!, [:PID, :reward]], header = false)
end
#---- Exclude according to memory quiz
memory = filter(x -> x.trial_type == "memory-quiz", dat) # Select trials
memory = memory[:, filter(x -> !isequal(unique(memory[!, x]),
    [missing]), names(memory))] # Remove empty variables
memory.correct = memory.chosen_table .== memory.answer # Compute correct

memory = combine(groupby(memory, [:PID]), :correct => sum => :correct, :correct => length => :n)
memory.p = 1 .- cdf.(Binomial.(memory.n, 0.25), memory.correct)

pre = nrow(memory)
mem_exc = filter(x -> x.p > 0.05, memory)
filter!(x -> x.p < 0.05, memory)

println("\nRemoved ", pre - nrow(memory), " participants for low memory score")
println(mem_exc.PID)
println("\nAverage memory score remaining: ", mean(memory.correct ./ memory.n))

dat = filter(x -> x.PID in memory.PID, dat) # Exclude from data

#---- Exclude according to deck choices
# Separate learning phase
learn = filter(x -> !ismissing(x.category) &&
    x.category in ["object_choice", "deck_choice", "too-quick", "focus-blur"], dat)

# Remove aborted trials
filter!(x -> !(x.category == "object_choice" && x.rt == "null"), learn) # Remove no response object choice trials

bad_indx = []
for ii in nrow(learn):-1:1
    if learn[ii, :category] == "deck_choice" && learn[ii, :rt] == "null"
        push!(bad_indx, ii, ii - 1)
    elseif learn[ii, :category] == "too-quick" || learn[ii, :category] == "focus-blur"
        push!(bad_indx, ii, ii - 1)
        if learn[ii-1, :category] == "deck_choice"
            push!(bad_indx, ii - 2)
        end
    end
end

learn = learn[filter(x -> !(x in bad_indx), 1:nrow(learn)), :]

# Remove empty columns
learn = learn[:, filter(x -> !isequal(unique(learn[!, x]), [missing]), names(learn))]

# Make RT numeric, on seconds scale
learn.rt = (x -> parse(Float64, x)).(learn.rt) / 1000

# Transform dataframe to have one line per trial: merge table and deck choice lines
wl = filter(x -> x.category == "object_choice", learn)

rename!(wl, :rt => :table_rt)

wl[!, :rt] .= 0.

for ii in 1:nrow(wl)
    id = wl[ii, :PID]
    sess = wl[ii, :sess]
    bl = wl[ii, :block]
    ind = wl[ii, :trial_index]

    # Candidate row to pull data from - has same Participand ID, same block, matching trial index
    cand = filter(x -> x.PID == id && x.sess == sess && x.block == bl && x.trial_index == ind + 3, learn)
    if nrow(cand) > 0
        for jj in [:LCard, :RCard, :rt, :chosen_deck_side, :chosen_deck, :reward]
            wl[ii, jj] = cand[!, jj][1]
        end
    end
end

rename!(wl, :rt => :deck_rt);

wl.trial = reduce(vcat, [1:nrow(x) for x in groupby(wl, [:PID, :sess, :block])])

# Count which deck was chosen
decks = combine(groupby(wl, [:PID, :sess, :block, :chosen_object]),
    :chosen_deck => (x -> minimum(skipmissing(x))) => :min_deck)
wl = innerjoin(wl, decks, on = [:PID, :sess, :block, :chosen_object])
wl.chosen_deck_N = [x.chosen_deck == x.min_deck ? 1 : 2 for x in eachrow(wl)]

wl.running_table_n = [x.chosen_object + 4 * (x.block) for x in eachrow(wl)]

# For each subject and game, compute the absolute deviation from choosing both decks 50:50
deck_s = combine(groupby(wl, [:PID, :running_table_n]),
    :chosen_deck_N => (x -> abs(0.5 - mean(x .== 1))) => :ratio,
    :chosen_deck_N => length => :n_trials)

# Multiply each table's ratio by the number of trials for that table
deck_s.wrat = deck_s.ratio .* deck_s.n_trials

# Compute weighted average
deck_s = combine(groupby(deck_s, :PID), [:wrat, :n_trials] => ((a,b) -> sum(a) / sum(b)) => :wrat)

# Remove subjects by exclusion criterion
pre = nrow(deck_s)
deck_exc = filter(x -> x.wrat > deck_ratio_deviation, deck_s)
filter!(x -> x.wrat <= deck_ratio_deviation, deck_s)
println("\nRemoved ", pre - nrow(deck_s), " participants for low entropy deck choices")
println(deck_exc.PID)

# Remove subjects from data
filter!(x -> x.PID in deck_s.PID, dat)

# Print list of PIDs for next session
function completed_next(x)
    next_sess = filter(y -> occursin("sess" * string(parse(Int, sess) + 1), y) &&
        !occursin("plan", y) &&
        !occursin("int", y) &&
        occursin("csv", y), readdir("../Data/"* subdir* "/"))

    sum((y -> occursin(x, y)).(next_sess)) > 0
end

future = filter(x -> completed_next(x), unique(dat.PID))

println("\nValid Ps this session: n=", length(unique(dat.PID)))

println("\nPIDs for next session:")
next = filter(x -> !(x in future),unique(dat.PID))
println("\nn=", length(next))
(x -> println(x)).(next)


# Supertype for all Firm Identifiers
abstract type FirmIdentifier end

# Some identifiers are essentially strings, this is a super class for all of those
abstract type FirmIdentifierString <: FirmIdentifier end

# Reamining are similar to integers, this deals with those cases
abstract type FirmIdentifierInt <: FirmIdentifier end

# Technically, there is only 1 Cusip, however, WRDS considers two types (NCusip and Cusip)
# which are basicaly just old and current versions, since other than the actual
# numbers, these are exactly the samne, this abstract type deals with all of those
# so I can have common functions
abstract type CusipAll <: FirmIdentifierString end

# checksum calculator for Cusip
function luhn_checksum(s)
    if length(s) != 8
        error("Length must be 8 digits")
    end
    tot = 0
    for (i, v) in enumerate(s)
        n = if '0' ≤ v ≤ '9'
            parse(Int, v)
        elseif 'A' ≤ v ≤ 'Z'
            Int(v) - 65 + 10
        elseif 'a' ≤ v ≤ 'z'
            Int(v) - 97 + 10
        elseif v == '*'
            36
        elseif v == '@'
            37
        elseif v == '#'
            38
        end
        if iseven(i)
            n *= 2
        end
        tot += floor(n / 10) + n % 10
    end
    (10 - tot % 10) % 10 |> Int
end

"""
GVKey is the primary identifier in the Compustat universe
It is a string, though only has numeric values
"""
struct GVKey <: FirmIdentifierString
    val::String
    function GVKey(s::String)
        if length(s) > 6
            error("GVKey must be 6 characters long")
        end
        if occursin(r"\D", s)
            error("GVKey can only contain numeric characters")
        end
        new(lpad(s, 6, "0"))
    end
end

GVKey(n::Real) = GVKey(string(Int(n)))

"""
CIK is a common identifier outside of WRDS
It is a string, though has numeric values
"""
struct CIK <: FirmIdentifierString
    val::String
    function CIK(s::String)
        if length(s) > 10
            error("CIK must be 10 characters long")
        end
        if occursin(r"\D", s)
            error("CIK can only contain numeric characters")
        end
        new(lpad(s, 10, "0"))
    end
end

CIK(n::Real) = CIK(string(Int(n)))

"""
Cusip is a common identifier within and outside of WRDS
WRDS tracks older Cusips as NCusip
all Cusips are made up of 3 parts, issuer (6 characters)
issue (2 characters), and a checksum
most databases in WRDS only use the 8 characters
"""
struct Cusip <: CusipAll
    issuer::String
    issue::String
    checksum::Int
    function Cusip(issuer::AbstractString, issue::AbstractString, checksum::Int)
        if length(issuer) != 6
            error("Issuer identification must be 6 characters")
        end
        if length(issue) != 2
            error("Issue must be 2 characters")
        end
        if (!)(0 ≤ checksum ≤ 9)
            error("Checksum must be between 0 and 9 (inclusive)")
        end
        if luhn_checksum(issuer * issue) != checksum
            error("Invalid Cusip or Checksum")
        end
        new(String(issuer), String(issue), checksum)
    end
end

function Cusip(s::AbstractString)
    if length(s) == 9
        Cusip(s[1:6], s[7:8], parse(Int, s[9]))
    elseif length(s) == 8
        Cusip(s[1:6], s[7:8], luhn_checksum(s))
    else
        error("Cusip must be 8 or 9 characters")
    end
end

Cusip(issuer::AbstractString, issue::AbstractString) = Cusip(issuer, issue, luhn_checksum(issuer * issue))

"""
See note on Cusip
"""
struct NCusip <: CusipAll
    issuer::String
    issue::String
    checksum::Int
    function NCusip(issuer::AbstractString, issue::AbstractString, checksum::Int)
        if length(issuer) != 6
            error("Issuer identification must be 6 characters")
        end
        if length(issue) != 2
            error("Issue must be 2 characters")
        end
        if (!)(0 ≤ checksum ≤ 9)
            error("Checksum must be between 0 and 9 (inclusive)")
        end
        if luhn_checksum(issuer * issue) != checksum
            error("Invalid NCusip or Checksum")
        end
        new(string(issuer), string(issue), checksum)
    end
end

function NCusip(s::AbstractString)
    if length(s) == 9
        NCusip(s[1:6], s[7:8], parse(Int, s[9]))
    elseif length(s) == 8
        NCusip(s[1:6], s[7:8], luhn_checksum(s))
    else
        error("NCusip must be 8 or 9 characters")
    end
end

NCusip(issuer::AbstractString, issue::AbstractString) = NCusip(issuer, issue, luhn_checksum(issuer * issue))

Cusip(x::NCusip) = Cusip(x.issuer, x.issue, x.checksum)
NCusip(x::Cusip) = NCusip(x.issuer, x.issue, x.checksum)

"""
Ticker here refers to the ticker that is on the NYSE (not to be confused with IbesTicker)
While a common identifier outside of WRDS, this is not recommended since they change often
"""
struct Ticker <: FirmIdentifierString
    val::String
end

Ticker(s::AbstractString) = Ticker(string(s))

"""
IbesTicker is the primary identifier in the IBES universe
"""
struct IbesTicker <: FirmIdentifierString
    val::String
end

IbesTicker(s::AbstractString) = IbesTicker(string(s))

"""
Permno is the primary identifier of securities in the CRSP universe, it is also
one of the most common methods of linking between databases since it is easy to
find links to IBES, Compustat (GVKey), and Cusip.
"""
struct Permno <: FirmIdentifierInt
    val::Int
end

Permno(x::Real) = Permno(Int(x))

Permno(x::Permno) = x
Cusip(x::Cusip) = x
CIK(x::CIK) = x
GVKey(x::GVKey) = x
NCusip(x::NCusip) = x
Ticker(x::Ticker) = x
IbesTicker(x::IbesTicker) = x

Permno(x::Missing) = x
Cusip(x::Missing) = x
CIK(x::Missing) = x
GVKey(x::Missing) = x
NCusip(x::Missing) = x
Ticker(x::Missing) = x
IbesTicker(x::Missing) = x

######################################################################################
# As long as convert is specified for an identifier, all of the other functions here
# should work automatically work (as long as the type fits into string or integer)
######################################################################################

# Used for Permno
Base.convert(::Type{T}, x::K) where {T<:Real,K<:FirmIdentifierInt} = convert(T, x.val)

# Used for Permno, GVKey, and CIK
Base.convert(::Type{K}, x::Real) where {K<:FirmIdentifier} = K(x)

# Used for GVKey, CIK, Ticker, and IbesTicker
Base.convert(::Type{T}, x::K) where {T<:AbstractString,K<:FirmIdentifierString} = convert(T, x.val)

# Special case for GVKey and CIK
Base.convert(::Type{T}, x::GVKey) where {T<:Real} = parse(T, x.val)
Base.convert(::Type{T}, x::CIK) where {T<:Real} = parse(T, x.val)

# Used for GVKey, CIK, Ticker, IbesTicker, Cusip, and NCusip
Base.convert(::Type{K}, x::AbstractString) where {K<:FirmIdentifierString} = K(x)

# Used for Cusip and NCusip
Base.convert(::Type{T}, x::K) where {T<:AbstractString,K<:CusipAll} = convert(T, "$(x.issuer)$(x.issue)")

# Comparisons
# These all convert the non-identifier into an identifier before back to the base value
# This ensures consistency in the comparison, no issues with CIK being an Int in one place and a string in another

# I am not sure these are actually useful, they were made to try and make joins possible but that is handled by hash
Base.:(==)(id::K, s::T) where {T<:AbstractString, K<:FirmIdentifier} = convert(T, id) == convert(T, convert(K, s))
Base.:(==)(s::T, id::K) where {T<:AbstractString, K<:FirmIdentifier} = convert(T, convert(K, s)) == convert(T, id)

# isless is valuable for sorting
Base.isless(id::K, s::T) where {T<:AbstractString, K<:FirmIdentifier} = isless(convert(T, id), convert(T, convert(K, s)))
Base.isless(s::T, id::K) where {T<:AbstractString, K<:FirmIdentifier} = isless(convert(T, convert(K, s)), convert(T, id))
Base.isless(id1::FirmIdentifier, id2::FirmIdentifier) = isless(value(id1), value(id2))

# String based identifiers

Base.show(io::IOContext, id::FirmIdentifierString) = show(io, convert(String, id))
# Base.print(io::IOContext, x::GVKey) = print(io, x.val)
Base.string(id::FirmIdentifierString) = convert(String, id)

# tryparse is useful for csv
Base.tryparse(::Type{T}, s::AbstractString) where {T<:FirmIdentifierString} = try T(s) catch nothing end
value(id::FirmIdentifierString) = convert(String, id)

# Special "value" parameter for CusipAll so it can return 9 characters
value(x::CusipAll, l::Int=8) = "$(x.issuer)$(x.issue)$(x.checksum)"[1:l]

Base.promote_rule(::Type{T}, ::Type{K}) where {T<:FirmIdentifierString,K<:AbstractString} = K

Base.hash(id::FirmIdentifierString) = hash(convert(String, id))

# Integer based identifiers

Base.show(io::IOContext, id::FirmIdentifierInt) = show(io, convert(Int, id))
Base.string(id::FirmIdentifierInt) = string(convert(Int, id))
value(id::FirmIdentifierInt) = convert(Int, id)

Base.promote_rule(::Type{T}, ::Type{K}) where {T<:FirmIdentifierInt,K<:Real} = K
Base.tryparse(::Type{T}, s::AbstractString) where {T<:FirmIdentifierInt} = try T(parse(Int, s)) catch nothing end
Base.hash(id::FirmIdentifierInt) = hash(convert(Int, id))

# to prevent errors for missing data
value(x::Missing) = missing

######################################################################################
# Future notes: One thing that would be nice is if I could specify `isequal` for
# items that are "number like", for example, it would be nice if
# GVKey("000015") == 15 returned true (which is easy) and the has values were the same
# (which I cannot figure out how to do). As far as I can tell, I can specify the hash
# value as either a String or an Integer, but not both

# Another future task is to make the has value of a tuple to be the same as the base
# version, right now this works well for normal joins but not well for groupby,
# which of course is what my join function relies on
######################################################################################


# index.jl - tim.sterne.weiler@utoronto.ca, 1/28/16

abstract type SeqLibrary end

struct GraphLib <: SeqLibrary
   offset::Vector{CoordInt}
   names::Vector{GeneName}
   info::Vector{GeneInfo}
   lengths::Vector{Float64}
   graphs::Vector{SpliceGraph}
   edges::Edges
   index::FMIndex
   sorted::Bool
   kmer::Int64
end

function build_chrom_dict( ref::RefSet )
   ret = Dict{SeqName,Vector{GeneName}}() # refgenomeseq->geneid[]
   for g in keys(ref)
      chrom = ref[g].info.name
      if !haskey(ret, chrom)
         ret[chrom] = Vector{GeneName}()
      end
      push!(ret[chrom], g)
   end
   ret
end

function twobit_enc(seq)
   len = length(seq)
   ret = IntVector{2,UInt8}(len)
   for i in 1:len
      val = isambiguous(seq[i]) ? 0x00 : convert(UInt8, trailing_zeros(seq[i]))
      ret[i] = val
   end
   ret
end

function load_fasta( fhIter; verbose=false )
   seq = SGSequence(mutable=false)
   offset = Int[]
   names = SeqName[]
   for r in fhIter
      immutable!(r.seq)
      push!(names, r.name)
      push!(offset, length(seq)+1)
      println( STDERR, r )
      @time seq *= r.seq
   end
   seq, offset, names
end

function single_genome_index!( fhIter; verbose=false )
   seq,offset,names = load_fasta( fhIter )
   @time fm = FMIndex(twobit_enc(seq), 4, r=4, program=:SuffixArrays, mmap=true)
   println( STDERR, "Finished building Index..." )

   SeqLib(seq, offset, names, fm, false)
end

function trans_index!( fhIter, ref::RefSet; kmer=9 )
   seqdic  = build_chrom_dict( ref )
   xcript  = dna""
   xoffset = Vector{UInt64}()
   xgenes  = Vector{GeneName}()
   xinfo   = Vector{GeneInfo}()
   xlength = Vector{Float64}()
   xgraph  = Vector{SpliceGraph}()
   # set up splice graphs
   runoffset = 0
   num = 0
   for r in fhIter
      name = String(r.data[r.identifier])
      haskey(seqdic, name) || continue
      #BioSequences.immutable!(r.seq)
      sg = SGSequence( r.data[r.sequence] )

      println(STDERR, "Building Splice Graphs for $( name ).." )
      @timer for g in seqdic[name]
         #println( STDERR, "Building $g Splice Graph..." )
         curgraph = SpliceGraph( ref[g], sg, kmer )
         xcript  *= curgraph.seq
         push!(xgraph, curgraph)
         push!(xgenes, g)
         push!(xinfo, ref[g].info )
         push!(xlength, ref[g].length )
         push!(xoffset, runoffset)
         runoffset += length(curgraph.seq)
         num += 1
      end
   end
   if num > 1
      println( STDERR, "Building full sg-index from $num genes..." )
      @time fm = FMIndex(twobit_enc(xcript), 4, r=1, program=:SuffixArrays, mmap=true)
   else
      error("ERROR: No genes in the GTF file matched chromosome names in the FASTA file!")
   end

   # clean up
   xcript = DNASequence()
   gc()

   println( STDERR, "Building edges.." )
   @time edges = build_edges( xgraph, kmer ) # TODO make variable kmer

   GraphLib( xoffset, xgenes, xinfo, xlength, xgraph, edges, fm, true, kmer )
end

function fasta_to_index( filename::String, ref::RefSet; kmer=9 )
   if isgzipped( filename )
      println(STDERR, "Decompressing and Indexing $filename...")
      to_open = open( filename ) |> x->ZlibInflateInputStream(x, reset_on_end=true)
   else
      println(STDERR, "Indexing $filename...")
      to_open = open( filename ) |> BufferedInputStream
   end
   # iterate through fasta entries
   index = @time trans_index!(FASTA.Reader( to_open ), ref, kmer=kmer)
   index
end


using PortAudio, SampledSignals, LibSndFile
using HTTP
using WAV
using Statistics
using Plots
using Images
using PyCall
using DSP

function spectronew(val)
    fs = 16000
    audio_test = SampleBuf(val, 16000)
    n = length(audio_test.data)
    nw = n÷25
    spec = DSP.spectrogram(audio_test.data[:], nw, nw÷10; fs=fs)
    png(
        Plots.heatmap(spec.time, spec.freq, pow2db.(spec.power), xaxis = false, yaxis=false, legend = :none, ticks=false, margins = -1.2Plots.cm, size=(28.5,29)),
        "testcase",
    )
    img = load("testcase.png")
    img = imresize(img, (28,28))
    img = reshape(convert(Array{Float32}, Gray.(img)), (28, 28, 1));
    return img
end

function sendsingledataup(data)
   r  = HTTP.post("http://localhost:8003/adddata", [], string(spectronew(data)))
   # println(String(r.body))
end

labeldict = Dict("left" => 0, "right" => 1, "up" => 2,  "down" => 3,""=>4)

function makeamove(mode)
    py"""
    import pyautogui as autogui
    def makemode(mode):
        #autogui.moveTo(200, 200, duration=1)
        if mode == 0:
            print('Mode: Left')
            #autogui.click(clicks=2, button='left')
            autogui.press('left')
        elif mode == 1:
            print('Mode: Right')
            autogui.press('right')
        elif mode == 2:
            print('Mode: Up')
            #autogui.click(clicks=2, button='up')
            autogui.press('up')
        elif mode == 3:
            print('Mode: Down')
            autogui.press('down')
        else:
            print('Mode: NA')
    """
    py"makemode"(mode)
end

py"""
import collections
import contextlib
import sys
import wave

import webrtcvad

def read_wave(path):
    with contextlib.closing(wave.open(path, 'rb')) as wf:
        num_channels = wf.getnchannels()
        assert num_channels == 1
        sample_width = wf.getsampwidth()
        assert sample_width == 2
        sample_rate = wf.getframerate()
        assert sample_rate in (8000, 16000, 32000, 48000)
        pcm_data = wf.readframes(wf.getnframes())
        return pcm_data, sample_rate

def write_wave(path, audio, sample_rate):
    with contextlib.closing(wave.open(path, 'wb')) as wf:
        wf.setnchannels(1)
        wf.setsampwidth(2)
        wf.setframerate(sample_rate)
        wf.writeframes(audio)

class Frame(object):
    def __init__(self, bytes, timestamp, duration):
        self.bytes = bytes
        self.timestamp = timestamp
        self.duration = duration

def frame_generator(frame_duration_ms, audio, sample_rate):
    n = int(sample_rate * (frame_duration_ms / 1000.0) * 2)
    offset = 0
    timestamp = 0.0
    duration = (float(n) / sample_rate) / 2.0
    while offset + n < len(audio):
        yield Frame(audio[offset:offset + n], timestamp, duration)
        timestamp += duration
        offset += n

def vad_collector(sample_rate, frame_duration_ms,
                  padding_duration_ms, vad, frames):
    num_padding_frames = int(padding_duration_ms / frame_duration_ms)
    ring_buffer = collections.deque(maxlen=num_padding_frames)
    triggered = False

    voiced_frames = []
    for frame in frames:
        is_speech = vad.is_speech(frame.bytes, sample_rate)

        #sys.stdout.write('1' if is_speech else '0')
        if not triggered:
            ring_buffer.append((frame, is_speech))
            num_voiced = len([f for f, speech in ring_buffer if speech])
            if num_voiced > 0.9 * ring_buffer.maxlen:
                triggered = True
                # sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,))
                for f, s in ring_buffer:
                    voiced_frames.append(f)
                ring_buffer.clear()
        else:
            voiced_frames.append(frame)
            ring_buffer.append((frame, is_speech))
            num_unvoiced = len([f for f, speech in ring_buffer if not speech])
            if num_unvoiced > 0.9 * ring_buffer.maxlen:
                # sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
                triggered = False
                yield b''.join([f.bytes for f in voiced_frames])
                ring_buffer.clear()
                voiced_frames = []
    # if triggered:
        # print("Here I arrive")
        # sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
    #sys.stdout.write('\n')
    if voiced_frames:
        yield b''.join([f.bytes for f in voiced_frames])

def doyourvaejobplease(path):
    audio, sample_rate = read_wave(path)
    vad = webrtcvad.Vad(int(1))
    frames = frame_generator(30, audio, sample_rate)
    frames = list(frames)
    segments = vad_collector(sample_rate, 30, 300, vad, frames)
    for i, segment in enumerate(segments):
        path = './audios/chunk-%002d.wav' % (i,)
        write_wave(path, segment, sample_rate)
"""

using Glob
while true
    stream = PortAudioStream("default", 1, 0)
    buf = read(stream, 2s).data[:]
    wavwrite(buf,  "test8final.wav", Fs=48000, nbits=16,  compression=WAVE_FORMAT_PCM)
    py"doyourvaejobplease"("test8final.wav")
    close(stream)
    listofwavs = glob("*.wav","./audios")
    interestingdata = map(x->wavread(x)[1][:], listofwavs)
    for i in interestingdata
        datara = ceil(Int, length(i)/16000)
        zerosadd =  datara*16000 - length(i)
        zeroestobeadded = zeros(Float64, zerosadd)
        i = vcat(i, zeroestobeadded)
        batch = map(x->i[16000*(x-1)+1:16000*x] ,1:datara)
    	for j in batch
            sendsingledataup(j)
        end
    end

    # # rm("audios")
    map(x->rm(x), listofwavs)
    interestingdata = nothing
    listofwavs = nothing
    buf = nothing
    r = HTTP.get("http://localhost:8003/digitreg", [])
    mode = labeldict[String(r.body)]
    makeamove(mode)
    #print(varinfo())
    sleep(0.1)
end


    safe_email_tlds = ("org", "com", "net")
    free_email_domains = ("gmail.com", "yahoo.com", "hotmail.com")
    tlds = ("com", "com", "com", "com", "com", "com", "biz", "info", "net", "org")

    uri_pages = (
    "index", "home", "search", "main", "post", "homepage", "category", "register", "login", "faq", "about", "terms",
    "privacy", "author")
    uri_paths = (
    "app", "main", "wp-content", "search", "category", "tag", "categories", "tags", "blog", "posts", "list", "explore")
    uri_extensions = (".html", ".html", ".html", ".htm", ".htm", ".php", ".php", ".jsp", ".asp")

    user_name_formats = (
        "$(last_name()).$(first_name())",
        "$(first_name()).$(last_name())",
        "$(first_name())##",
        "?$(last_name())",
    )

    email()= email_formats[rand(1:length(email_formats))]

    user_name()= bothify(user_name_formats[rand(1:length(user_name_formats))])

    safe_email()= "$(user_name())@example.$(safe_email_tlds[rand(1:length(safe_email_tlds))])"

    free_email()= "$(user_name())@$(free_email_domain())"

    domain_suffix()= tlds[rand(1:length(tlds))]

    domain_word()=
       ( company_elements = split(company(), [',', ' ','-']);
        sal = "";
        for i in company_elements
                   if length(i)>0
                       sal*="$(i[1])"
                   end
              end;

        lowercase(sal) )

    domain_name()="$(domain_word()).$(domain_suffix())"

    company_email()= "$(user_name())@$(domain_name())"

    free_email_domain() = free_email_domains[rand(1:length(free_email_domains))]

    url()=url_formats[rand(1:length(url_formats))]

    ipv4()= "$(rand(172:255)).$(rand(0:255)).$(rand(0:255)).$(rand(0:255))"

    ipv6()= (res = "";
        for i = 1:8
            res*= "$(hex(rand(4100:65535)))"
            if i < 8
                res*= ":"
            end
        end;res)

    mac_address()=
        (res = "";
        for i = 1:6
            res*= "$(hex(rand(0x00:0xff)))"
            i < 6 && (res*= ":")
        end;res)

    uri_page()= uri_pages[rand(1:length(uri_pages))]

    uri_path(deep="None")=
        (if deep =="None"
            deep = rand(1:3)
        end; sa="";
            for i =0:deep
               sa*= uri_paths[rand(1:length(uri_paths))]
                i < deep && (sa*= "/")
            end; sa )

    uri_extension()= uri_extensions[rand(1:length(uri_extensions))]

    uri()= uri_formats[rand(1:length(uri_formats))]

    #=slug(value="None")=
        if value == "None"
            value = Lorem.text(20)
            value=#

    image_url(width="None", height="None")=
       image_placeholder_services[rand(1:length(image_placeholder_services))]

    email_formats = (
        "$(user_name())@$(domain_name())",
        "$(user_name())@$(free_email_domain())",
    )
    url_formats = (
        "http://www.$(domain_name())/",
        "http://$(domain_name())/",
    )
    uri_formats = (
        "$(url())",
        "$(url())$(uri_page())/",
        "$(url())$(uri_page())$(uri_extension())",
        "$(url())$(uri_path())/$(uri_page())/",
        "$(url())$(uri_path())/$(uri_page())$(uri_extension())",
    )
    image_placeholder_services = (
        "http://placekitten.com/$(rand(1:1024))/$(rand(1:1024))",
        "http://placehold.it/$(rand(1:1024))x$(rand(1:1024))",
        "http://www.lorempixum.com/$(rand(1:1024))/$(rand(1:1024))",
        "http://dummyimage.com/$(rand(1:1024))x$(rand(1:1024))",
     )