#!/usr/bin/python3

import os
import numpy as np
import pandas as pd

import statistical as sl
import input_output as io
import importlib
#importlib.reload(statistical)
#importlib.reload(input_output)

filedir = '/home/anja/team10/team10/data/'     # has to be defined by the user
filenames = [f for f in os.listdir(filedir) if os.path.isfile(os.path.join(filedir, f))]
outdir = '/home/anja/team10/team10/data/'       #define output directory
threshv = 1.0e-5

# read expec.t
io.read_in_np(filedir, filenames='expec.t')
io.read_in_pd(filedir, filenames='expec.t')

# first overview of the dataset
io.first_analysis(filedir, filenames='expec.t')

# plot the dataset
sl.plot_file(df = io.first_analysis(filedir,filenames='expec.t'))

#save plot
plot_pd = sl.plot_file(df = io.first_analysis(filedir,filenames='expec.t'))
io.save_plots(outdir, outputfile = plot_pd, filenames = 'expec.t', title = 'plot_pd')

# eliminate columns based on the variance: if the variance of the values
# in a column is below a given threshold, that column is discarded; and define as new file {}_new
#column variance of the data set; define variance filter:

sl.stat_sign(threshv, df_to_filter=io.read_in_pd(filedir,filenames='expec.t'))

# plot new dataset
data_var = sl.stat_sign(threshv, df_to_filter=io.read_in_pd(filedir,filenames='expec.t'))
sl.plot_file_var(data_var, x=data_var ['time'], y=data_var['<z>'])

# save new plot
plot_pd_var = sl.plot_file_var(data_var, x=data_var['time'], y=data_var['<z>'])
io.save_plots(outdir, outputfile = plot_pd_var, filenames = 'expec.t', title = 'plot_pd_var')


## Read in file npop.t and analyze correlations in the data

# read file npop.t
io.read_in_np(filedir, filenames='npop.t')
io.read_in_pd(filedir, filenames='npop.t')

# first overview of the dataset
io.first_analysis(filedir, filenames='npop.t')

# show variances of the columns and discard all columns /
# with variance below a set threshold - we can consider them as constant

pd_file_var = sl.stat_sign(threshv, df_to_filter=io.read_in_pd(filedir,filenames='npop.t'))

# transform dataframe from wide format into long format
file_trans = sl.transform(tf_file=pd_file_var, stubnames=['MO'], i='time', j='measurement')

# plot transformed dataset and save plot as .pdf
file_trans_plot = sl.plot_file(df=file_trans, x='time', y='MO', hue='measurement')
io.save_plots(outdir, outputfile= file_trans_plot, filenames = 'npop.t', title = 'file_trans_plot')

#print and plot correlation matrix (pearson) and skip time column
corr_mat = sl.corr_mat(data_for_corr = file_trans)

#save corr matrix as csv
io.save_csv(outdir, outputfile= corr_mat, filenames = 'npop.t')

#extract upper triangle of the matrix
mat_triu = sl.corr_mat_triu (mat_for_triu= corr_mat)

#save the plot
io.save_plots(outdir, outputfile = mat_triu, filenames = 'npop.t', title = 'mat_triu')

#!/usr/bin/python3

import os
import numpy as np
import pandas as pd

import statistical as sl
import input_output as io
import importlib
#importlib.reload(statistical)
#importlib.reload(input_output)

filedir = '/home/anja/team10/team10/data/'     # has to be defined by the user
filenames = [f for f in os.listdir(filedir) if os.path.isfile(os.path.join(filedir, f))]
outdir = '/home/anja/team10/team10/data/'       #define output directory
threshv = 1.0e-5

# read expec.t
io.read_in_np(filedir, filenames='expec.t')
io.read_in_pd(filedir, filenames='expec.t')

# first overview of the dataset
io.first_analysis(filedir, filenames='expec.t')

# plot the dataset
sl.plot_file(df = io.first_analysis(filedir,filenames='expec.t'))

#save plot
plot_pd = sl.plot_file(df = io.first_analysis(filedir,filenames='expec.t'))
io.save_plots(outdir, outputfile = plot_pd, filenames = 'expec.t', title = 'plot_pd')

# eliminate columns based on the variance: if the variance of the values
# in a column is below a given threshold, that column is discarded; and define as new file {}_new
#column variance of the data set; define variance filter:

sl.stat_sign(threshv, df_to_filter=io.read_in_pd(filedir,filenames='expec.t'))

# plot new dataset
data_var = sl.stat_sign(threshv, df_to_filter=io.read_in_pd(filedir,filenames='expec.t'))
sl.plot_file_var(data_var, x=data_var ['time'], y=data_var['<z>'])

# save new plot
plot_pd_var = sl.plot_file_var(data_var, x=data_var['time'], y=data_var['<z>'])
io.save_plots(outdir, outputfile = plot_pd_var, filenames = 'expec.t', title = 'plot_pd_var')


## Read in file npop.t and analyze correlations in the data

# read file npop.t
io.read_in_np(filedir, filenames='npop.t')
io.read_in_pd(filedir, filenames='npop.t')

# first overview of the dataset
io.first_analysis(filedir, filenames='npop.t')

# show variances of the columns and discard all columns /
# with variance below a set threshold - we can consider them as constant

pd_file_var = sl.stat_sign(threshv, df_to_filter=io.read_in_pd(filedir,filenames='npop.t'))

# transform dataframe from wide format into long format
file_trans = sl.transform(tf_file=pd_file_var, stubnames=['MO'], i='time', j='measurement')

# plot transformed dataset and save plot as .pdf
file_trans_plot = sl.plot_file(df=file_trans, x='time', y='MO', hue='measurement')
io.save_plots(outdir, outputfile= file_trans_plot, filenames = 'npop.t', title = 'file_trans_plot')

#print and plot correlation matrix (pearson) and skip time column
corr_mat = sl.corr_mat(data_for_corr = file_trans)

#save corr matrix as csv
io.save_csv(outdir, outputfile= corr_mat, filenames = 'npop.t')

#extract upper triangle of the matrix
mat_triu = sl.corr_mat_triu (mat_for_triu= corr_mat)

#save the plot
io.save_plots(outdir, outputfile = mat_triu, filenames = 'npop.t', title = 'mat_triu')

#!/usr/bin/python3 #importlib.reload(statistical) #importlib.reload(input_output) # has to be defined by the user #define output directory # read expec.t # first overview of the dataset # plot the dataset #save plot # eliminate columns based on the variance: if the variance of the values # in a column is below a given threshold, that column is discarded; and define as new file {}_new #column variance of the data set; define variance filter: # plot new dataset # save new plot ## Read in file npop.t and analyze correlations in the data # read file npop.t # first overview of the dataset # show variances of the columns and discard all columns / # with variance below a set threshold - we can consider them as constant # transform dataframe from wide format into long format # plot transformed dataset and save plot as .pdf #print and plot correlation matrix (pearson) and skip time column #save corr matrix as csv #extract upper triangle of the matrix #save the plot