import itertools
import numpy as np
# from pyjavaproperties import Properties
from vaex.ext import jprops
import collections
import functools
from vaex.ui.qt import *
import vaex.dataset
import vaex.ui.plot_windows
import logging as logging
import vaex.ui.qt as dialogs
import vaex.execution
import vaex.kld
from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
# from matplotlib.backends.backend_qt4agg import NavigationToolbar2QTAgg as NavigationToolbar
from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT as NavigationToolbar
import matplotlib.pyplot as plt
import scipy.stats

logger = logging.getLogger("vaex.ranking")

# since we do many columns at once, a smallar buffer will lead to more resposiveness in the gui
buffer_size = 1e6

def unique_column_names(dataset):
    # return list(set(dataset.column_names) | set(dataset.virtual_columns.keys()))
    return dataset.get_column_names(virtual=True)

testing = False

class PlotDialog(QtGui.QDialog):
    def __init__(self, parent, width=5, height=4, dpi=100, **options):
        super(PlotDialog, self).__init__()
        self.parent_widget = parent
        self.figure = plt.figure()
        self.canvas = FigureCanvas(self.figure)
        self.toolbar = NavigationToolbar(self.canvas, self)

        self.layout = QtGui.QVBoxLayout()
        self.layout.addWidget(self.toolbar)
        self.layout.addWidget(self.canvas)
        self.setLayout(self.layout)
        self.figure.canvas.mpl_connect('motion_notify_event', self._on_mouse_motion)
        self.figure.canvas.mpl_connect('button_press_event', self._on_mouse_button)

        # self.plot()

    def _on_mouse_motion(self, event):
        if event.xdata is not None and event.ydata is not None:
            self.on_mouse_motion(event)

    def _on_mouse_button(self, event):
        if event.xdata is not None and event.ydata is not None:
            self.on_mouse_button(event)

    def on_mouse_button(self, event):
        pass

    def on_mouse_motion(self, event):
        label, x, y = self.get_tooltip(event)
        self.set_tooltip(x, y, label)

    def set_tooltip(self, x, y, label):
        y = self.canvas.height() - 1 - y
        print(("motion", label, x, y))
        if label:
            print(("self.canvas.x/y()", self.canvas.geometry().x(), self.canvas.geometry().y()))
            print(("self.pos.x/y()", self.canvas.pos().x() + self.pos().x(), self.canvas.pos().y() + self.pos().y()))
            point = QtCore.QPoint(x + self.canvas.x() + self.pos().x(), y + self.canvas.y() + self.pos().y())
            QtGui.QToolTip.showText(point, label)

    def plot(self):
        pass

class RankPlot(PlotDialog):
    def __init__(self, parent, table):
        super(RankPlot, self).__init__(parent)
        self.table = table

        self.button_update_data = QtGui.QPushButton('Update data')
        self.button_update_data.clicked.connect(self.update_data)
        self.layout.addWidget(self.button_update_data)

        grid_layout = QtGui.QGridLayout()
        grid_layout.setColumnStretch(2, 1)
        grid_layout.setAlignment(QtCore.Qt.AlignTop)
        grid_layout.setSpacing(0)
        grid_layout.setContentsMargins(0, 0, 0, 0)
        self.layout.addLayout(grid_layout)

        row = 0
        self.options = ["mutual_information", "rank(mutual_information)", "correlation_coefficient", "rank(correlation_coefficient)", "abs(correlation_coefficient)", "rank(abs(correlation_coefficient))"]
        self.expression_x = "mutual_information"
        self.expression_y = "correlation_coefficient"
        self.codeline_x = Codeline(self, "x", self.options,
                                   getter=attrgetter(self, "expression_x"),
                                   setter=attrsetter(self, "expression_x"),
                                   update=self.plot)
        self.codeline_y = Codeline(self, "y", self.options,
                                   getter=attrgetter(self, "expression_y"),
                                   setter=attrsetter(self, "expression_y"),
                                   update=self.plot)
        row = self.codeline_x.add_to_grid_layout(row, grid_layout)
        row = self.codeline_y.add_to_grid_layout(row, grid_layout)

        self.axes = self.figure.add_subplot(111)
        if 0:
            def onpick(event):
                thisline = event.artist
                xdata = thisline.get_xdata()
                ydata = thisline.get_ydata()
                index = event.ind
                print((index, xdata[index], ydata[index], self.pairs[index]))
                self.table.select_pair(self.pairs[index])
            self.figure.canvas.mpl_connect('pick_event', onpick)

        self.update_data()
        self.plot()

    def on_mouse_button(self, event):
        min_distance_index, x, y = self.find_nearest(event)
        print(("click", x, y))
        self.table.select_pair(self.pairs[min_distance_index])

        def update_tooltip():
            label, x, y = self.get_tooltip(event)
            self.set_tooltip(x, y, label)
        QtCore.QTimer.singleShot(100, update_tooltip)

    def find_nearest(self, event):
        transform = event.inaxes.transData.transform
        xy = transform(list(zip(self.x, self.y)))
        # print xy
        x, y = xy.T
        print(("event.x/y", event.x, event.y))
        distances = np.sqrt((x - event.x)**2 + (y - event.y)**2)
        min_distance_index = np.argmin(distances)
        return min_distance_index, x[min_distance_index], y[min_distance_index]

    def get_tooltip(self, event):
        min_distance_index, x, y = self.find_nearest(event)
        label = "-".join(self.pairs[min_distance_index])
        return label, x, y

    def update_data(self):
        self.variables = {}
        self.pairs = pairs = self.table.getSelected()
        mi = [self.table.qualities[pair] for pair in pairs]
        corr = [self.table.correlation_map[pair] for pair in pairs]
        self.variables["mutual_information"] = np.array(mi)
        self.variables["correlation_coefficient"] = np.array(corr)

    def plot(self):
        scope = {}
        scope.update(np.__dict__)
        scope.update(self.variables)
        scope["rank"] = scipy.stats.rankdata
        self.x = x = eval(self.expression_x, scope)
        self.y = y = eval(self.expression_y, scope)
        self.axes.cla()
        self.axes.plot(x, y, '.', picker=5)
        self.canvas.draw()

class ____RankingTableModel(QtCore.QAbstractTableModel):
    def __init__(self, dataset, dim=1, parent=None, *args):
        QtCore.QAbstractTableModel.__init__(self, parent, *args)
        self.dataset = dataset

        self.pairs = list(itertools.combinations(unique_column_names(self.dataset), dim))
        self.ranking = [None for pair in self.pairs]
        self.headers = ["subspace", "Mutual information", "MI ranking", "correlation", "MI ranking", 'selected']
        self.indices = list(range(len(self.pairs)))

    def rowCount(self, parent):
        return len(self.pairs)

    def columnCount(self, parent):
        return len(self.headers)

    def data(self, index, role):
        if not index.isValid():
            return None
        elif role != QtCore.Qt.DisplayRole:
            return None
        column = index.column()
        index = self.indices[index.row()]  # use sorted index
        if column == 0:
            return "-vs".join(self.pairs[index])
        if column == 1:
            rank = self.ranking[index]
            return "" if rank is None else str(rank)
        if column == 2:
            rank = self.ranking[index]
            return False if random.random() < 0.5 else True

    def headerData(self, index, orientation, role):
        if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
            return self.headers[index]
        if orientation == QtCore.Qt.Vertical and role == QtCore.Qt.DisplayRole:
            return str(index + 1)
        return None

    def sort(self, Ncol, order):
        """Sort table by given column number.
        """
        self.emit(QtCore.SIGNAL("layoutAboutToBeChanged()"))
        if Ncol == 0:
            print("by name")
            # get indices, sorted by pair name
            sortlist = list(zip(self.pairs, list(range(len(self.pairs)))))
            print(sortlist)
            sortlist.sort(key=operator.itemgetter(0))
            print(sortlist)
            self.indices = list(map(operator.itemgetter(1), sortlist))
            print((self.indices))
        if Ncol == 1:
            # get indices, sorted by ranking, or no sorting
            if None not in self.ranking:
                sortlist = list(zip(self.ranking, list(range(len(self.pairs)))))
                sortlist.sort(key=operator.itemgetter(0))
                self.indices = list(map(operator.itemgetter(1), sortlist))
            else:
                self.indices = list(range(len(self.pairs)))
            print((self.indices))
        if order == QtCore.Qt.DescendingOrder:
            self.indices.reverse()
        print((self.indices))
        self.emit(QtCore.SIGNAL("layoutChanged()"))

class SubspaceTable(QtGui.QTableWidget):
    def __init__(self, dialog, parent, mainPanel, dataset, pairs, dim, properties):
        self.dialog = dialog
        self.dim = dim
        if dim == 1:
            self.headers = ['', 'space', "min", "max", 'plot']
        else:
            self.headers = ['', 'space', "Mutual information", "Correlation", 'plot']
        self.properties = properties
        self.qualities = {}
        self.correlation_map = {}
        if testing:
            self.qualities = {key: np.random.random() for key in pairs}
            self.correlation_map = {key: np.random.normal() for key in pairs}

        # print ", ".join([""+("-".join(pair))+"" for pair in pairs])
        self.dataset = dataset
        self.filter_terms = []
        self.mainPanel = mainPanel
        self.pairs = list(pairs)  # list(itertools.combinations(self.dataset.column_names, dim))
        QtGui.QTableWidget.__init__(self, len(self.pairs), len(self.headers), parent)
        self.setSelectionBehavior(QtGui.QAbstractItemView.SelectRows)
        self.filter_mask = np.array([True for pair in pairs])
        self.selected_dict = {pair: self.properties.get(".".join(pair) + ".use", "True") == "True" for pair in pairs}
        # self.tableModel = RankingTableModel(self.dataset, dim, parent)
        # self.setModel(self.tableModel)
        # self.sortByColumn(0, QtCore.Qt.AscendingOrder)
        # self.setSortingEnabled(True)
        # self.pair_to_item = {}
        self.defaultFlags = QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsEditable
        # print self.properties._props
        if 1:
            # self.ranking = [None for pair in self.pairs]
            self.dim = dim
            self.setHorizontalHeaderLabels(self.headers)
            # self.setVerticalHeaderLabels(map(str, range(len(self.pairs))))
            self.fill_table()
            self.setSortingEnabled(True)
        self.queue_fill_table = vaex.ui.plot_windows.Queue("fill table", 200, self.fill_table)

    def pair_to_text(self, pair):
        return " ".join(map(str, pair))

    def select_pair(self, pair):
        # self.setSortingEnabled(False)
        index = self.pairs.index(pair)
        for i in range(self.rowCount()):
            item = self.item(i, 1)
            print((item.text(), self.pair_to_text(pair)))
            if item.text() == self.pair_to_text(pair):
                self.selectRow(i)
        # print index, self.visualRow(index)
        # self.selectRow(self.visualRow(index))
        # self.setSortingEnabled(True)

    def fill_table(self):
        # bug in qt? http://stackoverflow.com/questions/7960505/strange-qtablewidget-behavior-not-all-cells-populated-after-sorting-followed-b
        # fix: disable sorting, then enable again
        self.setSortingEnabled(False)
        self.checkboxes = []
        self.buttons = []
        pairs = [pair for pair, display in zip(self.pairs, self.filter_mask) if display]
        self.setRowCount(len(pairs))
        self.setVerticalHeaderLabels(list(map(str, list(range(len(pairs))))))
        for i in range(len(pairs)):
            pair = pairs[i]
            text = self.pair_to_text(pair)
            item = QtGui.QTableWidgetItem(text)
            self.setItem(i, 1, item)
            item.setFlags(self.defaultFlags)
            # item = QtGui.QTableWidgetItem()
            # item.setData(QtCore.Qt.DisplayRole, QtCore.QVariant(True))
            # item.setFlags(QtCore.Qt.ItemIsEditable | QtCore.Qt.ItemIsUserCheckable)
            checkbox = QtGui.QCheckBox(self)
            use_key = ".".join(map(str, pair)) + ".use"
            # if self.dim == 1 and use_key in self.properties._props:
            # #print use_key, eval(self.properties[use_key])
            # checkbox.setCheckState(QtCore.Qt.Checked if eval(self.properties[use_key]) else QtCore.Qt.Unchecked)
            # else:
            # checkbox.setCheckState(QtCore.Qt.Checked)
            print(("fill", pair, self.selected_dict[pair]))
            checkbox.setCheckState(QtCore.Qt.Checked if self.selected_dict[pair] else QtCore.Qt.Unchecked)
            self.checkboxes.append(checkbox)
            self.setCellWidget(i, 0, checkbox)

            def stateChanged(state, pair=pair):
                self.selected_dict[pair] = state == QtCore.Qt.Checked
                print(("set", pair, "to", self.selected_dict[pair]))
            checkbox.stateChanged.connect(stateChanged)

            if self.dim == 1:
                button = QtGui.QPushButton("plot: " + text, self)

                def plot(_ignore=None, pair=pair):
                    ranges = [self.dialog.range_map[k] for k in pair]
                    self.mainPanel.histogram(*pair, ranges=ranges)
                button.clicked.connect(plot)
                self.setCellWidget(i, 4, button)

                min_key = pair[0] + ".min"
                max_key = pair[0] + ".max"
                if 1:
                    # print "test", min_key
                    if min_key in self.properties:
                        item = QtGui.QTableWidgetItem()  # "%s"  % quality)
                        value = self.properties[min_key]
                        # print "it is in... and value =", value
                        item.setText("%s" % value)
                        item.setData(QtCore.Qt.DisplayRole, float(value))
                        item.setFlags(self.defaultFlags)
                        self.setItem(i, 2, item)

                    if max_key in self.properties:
                        value = self.properties[max_key]
                        item = QtGui.QTableWidgetItem()  # "%s"  % quality)
                        item.setText("%s" % value)
                        item.setData(QtCore.Qt.DisplayRole, float(value))
                        item.setFlags(self.defaultFlags)
                        self.setItem(i, 3, item)
            else:
                # print "quality", quality, qualities
                # row = self.pairs.index(pair)
                quality = self.qualities.get(pair)
                if quality is not None:
                    item = QtGui.QTableWidgetItem()  # "%s"  % quality)
                    item.setText("%s" % quality)
                    item.setData(QtCore.Qt.DisplayRole, float(quality))
                    item.setFlags(self.defaultFlags)
                    self.setItem(i, 2, item)
                correlation = self.correlation_map.get(pair)
                if correlation is not None:
                    item = QtGui.QTableWidgetItem()  # "%s"  % quality)
                    item.setText("%s" % correlation)
                    item.setData(QtCore.Qt.DisplayRole, float(correlation))
                    item.setFlags(self.defaultFlags)
                    self.setItem(i, 3, item)

            if self.dim == 2:
                button = QtGui.QPushButton("plot: " + text, self)

                def plot(_ignore=None, pair=pair):
                    ranges = [self.dialog.range_map[k] for k in pair]
                    self.mainPanel.plotxy(*pair, ranges=ranges)
                button.clicked.connect(
                    plot)
                self.setCellWidget(i, 4, button)
                self.buttons.append(button)  # keep ref count
            if self.dim == 3:
                button = QtGui.QPushButton("plot: " + text, self)

                def plot(_ignore=None, pair=pair):
                    ranges = [self.dialog.range_map[k] for k in pair]
                    self.mainPanel.plotxyz(*pair, ranges=ranges)
                button.clicked.connect(plot)
                self.setCellWidget(i, 4, button)
                self.buttons.append(button)  # keep ref count
            # self.setItem(i, 1, item)
        self.setSortingEnabled(True)

    def getSelected(self):
        selection = [checkbox.checkState() == QtCore.Qt.Checked for checkbox in self.checkboxes]
        selected_pairs = [pair for pair, selected in zip(self.pairs, selection) if selected]
        return selected_pairs

    def setQualities(self, pairs, qualities):
        self.qualities = {}
        for quality, pair in zip(qualities, pairs):
            self.qualities[pair] = quality
            # item = self.pair_to_item[pair]
            # print "quality", quality, qualities
            # row = self.pairs.index(pair)
            # item = QtGui.QTableWidgetItem()#"%s"  % quality)
            # item.setText("%s"  % quality)
            # item.setData(QtCore.Qt.DisplayRole, float(quality))
            # item.setFlags(self.defaultFlags)
            # self.setItem(row, 2, item)
        self.fill_table()

    def set_correlations(self, correlation_map):
        self.correlation_map = dict(correlation_map)
        self.fill_table()

    def get_range(self, pair):
        index = self.pairs.index(pair)
        mi = self.item(index, 2)
        ma = self.item(index, 3)
        if mi is None or ma is None:
            return None, None
        # print pair, mi, ma
        # print mi.data(QtCore.Qt.DisplayRole)
        mi = None if mi is None else float(mi.data(QtCore.Qt.DisplayRole))
        ma = None if ma is None else float(ma.data(QtCore.Qt.DisplayRole))
        # print "->", pair, mi, ma
        return mi, ma

    def setRanges(self, pairs, ranges):
        for (mi, ma), pair in zip(ranges, pairs):
            # item = self.pair_to_item[pair]
            row = self.pairs.index(pair)
            item = QtGui.QTableWidgetItem()  # "%s"  % quality)
            item.setText("%s" % mi)
            item.setData(QtCore.Qt.DisplayRole, float(mi))
            item.setFlags(self.defaultFlags)
            self.setItem(row, 2, item)
            item = QtGui.QTableWidgetItem()  # "%s"  % quality)
            item.setText("%s" % ma)
            item.setData(QtCore.Qt.DisplayRole, float(ma))
            item.setFlags(self.defaultFlags)
            self.setItem(row, 3, item)

    def deselect(self, pair):
        index = self.pairs.index(pair)
        print(("deselect", pair, index))
        checkbox = self.checkboxes[index]
        checkbox.setCheckState(QtCore.Qt.Unchecked)

    def select(self, pair):
        index = self.pairs.index(pair)
        print(("deselect", pair, index))
        checkbox = self.checkboxes[index]
        checkbox.setCheckState(QtCore.Qt.Checked)

    def setPairs(self, pairs):
        # selection = [checkbox.checkState() == QtCore.Qt.Checked for checkbox in self.checkboxes]
        # non_selected_pairs = [pair for pair, selected in zip(self.pairs, selection) if not selected]

        self.pairs = list(pairs)
        for pair in self.pairs:
            if pair not in self.selected_dict:
                self.selected_dict[pair] = self.properties.get(".".join(pair) + ".use", "True")
        self.filter_mask = np.array([True for pair in pairs])
        self.fill_table()
        # self.checkboxes = []
        # self.setRowCount(len(self.pairs))
        # self.setVerticalHeaderLabels(map(str, range(len(self.pairs))))
        # for i in range(len(self.pairs)):
        # text = " ".join(map(str, self.pairs[i]))
        # print text
        # item = QtGui.QTableWidgetItem(text)
        # item.setFlags(self.defaultFlags)
        # self.setItem(i, 1, item)
        # checkbox = self.checkboxes[i] #QtGui.QCheckBox(self)
        # if not (self.pairs[i] in non_selected_pairs):
        # checkbox.setCheckState(QtCore.Qt.Checked)
        # self.checkboxes.append(checkbox)
        # self.setCellWidget(i, 0, checkbox)
        # print self.checkboxes

    def set_filter_terms(self, filter_terms):
        def filter(pair):
            found = True
            for filter_term in filter_terms:
                found_term = False
                for expression in pair:
                    found_term = found_term or filter_term.lower() in expression.lower()
                found = found and found_term
            return found

        self.filter_terms = filter_terms
        # print list, filter, self.pairs
        self.filter_mask = np.array([filter(pair) for pair in self.pairs])
        self.queue_fill_table()
        # self.fill_table()

def joinpairs(pairs1d, pairsNd):
    previous = []
    for pair1d in pairs1d:
        subspacename = pair1d[0]  # tuple only has one element
        for pairNd in pairsNd:
            if subspacename not in pairNd:
                pair = pair1d + pairNd
                if sorted(pair) not in previous:
                    previous.append(sorted(pair))
                    # print previous
                    yield pair

class RankDialog(QtGui.QDialog):
    def __init__(self, dataset, parent, mainPanel, **options):
        super(RankDialog, self).__init__(parent)
        self.dataset = dataset
        self.mainPanel = mainPanel
        self.range_map = {}
        self.grid_size = int(options.get("grid_size", "32"))

        # print "options", options
        self.properties = collections.OrderedDict()  # Properties()
        if self.dataset.is_local():
            self.properties_path = os.path.splitext(self.dataset.path)[0] + ".properties"
        else:
            dir = os.path.join(vaex.utils.get_private_dir(), "ranking")
            if not os.path.exists(dir):
                os.makedirs(dir)
            server = self.dataset.server
            name = "%s_%s_%s_%s" % (server.hostname, server.port, server.base_path.replace("/", "_"), self.dataset.name)
            self.properties_path = os.path.join(dir, name + ".properties")

        self.properties_path = options.get("file", self.properties_path)
        if os.path.exists(self.properties_path):
            self.load_properties()
        else:
            pass
            # if not os.access(properties_path, os.W_OK):
            # dialog_error(self, "File access", "Cannot write to %r, so cannot save options" % properties_path)

        self.tabs = QtGui.QTabWidget(self)

        self.tab1d = QtGui.QWidget(self.tabs)
        self.table1d = SubspaceTable(self, self.tab1d, mainPanel, self.dataset, list(itertools.combinations(unique_column_names(self.dataset), 1)), 1, self.properties)

        self.subspaceTables = {}
        self.subspaceTabs = {}
        self.subspaceTables[1] = self.table1d
        self.subspaceTabs[1] = self.tab1d

        def onclick(dim=2):
            self.open(dim=dim)
        self.subspace2d = QtGui.QPushButton("create 2d subspaces", self.tab1d)
        self.subspace2d.clicked.connect(functools.partial(onclick, dim=2))

        self.get_ranges_menu = QtGui.QMenu()
        self.button_get_ranges = QtGui.QToolButton()
        self.button_get_ranges.setText("calculate min/max")
        self.button_get_ranges.setPopupMode(QtGui.QToolButton.InstantPopup)
        # self.button_get_ranges.setToolButtonStyle(QtCore.Qt.ToolButtonTextUnderIcon)
        self.get_ranges_menu = QtGui.QMenu()
        self.button_get_ranges.setMenu(self.get_ranges_menu)

        self.action_ranges_minmax = QtGui.QAction("absolute min/max", self)
        self.action_ranges_minmax_3sigma = QtGui.QAction("3 sigma clipping", self)
        self.get_ranges_menu.addAction(self.action_ranges_minmax)
        self.get_ranges_menu.addAction(self.action_ranges_minmax_3sigma)

        # self.button_get_ranges = QtGui.QToolButton(self.tab1d)
        # self.button_get_ranges.setText("calculate min/max")
        # self.button_get_ranges.setM
        self.action_ranges_minmax.triggered.connect(self.onCalculateMinMax)
        self.action_ranges_minmax_3sigma.triggered.connect(self.onCalculateMinMax3Sigma)

        self.button_store = QtGui.QToolButton(self.tab1d)
        self.button_store.setText("store")
        self.button_store.clicked.connect(self.onStore)

        self.actions_menu = QtGui.QMenu()
        self.button_actions = QtGui.QToolButton()
        self.button_actions.setText("Extra")
        self.button_actions.setPopupMode(QtGui.QToolButton.InstantPopup)
        self.button_actions.setMenu(self.actions_menu)

        self.action_select_all = QtGui.QAction("Select all", self)
        self.action_select_none = QtGui.QAction("Select none", self)
        self.action_remove_empty = QtGui.QAction("Remove empty columns", self)
        self.action_pca = QtGui.QAction("PCA transformation", self)
        self.action_select_all.triggered.connect(self.onSelectAll)
        self.action_select_none.triggered.connect(self.onSelectNone)
        self.action_remove_empty.triggered.connect(self.onRemoveEmpty)
        self.action_pca.triggered.connect(self.onPca)
        self.actions_menu.addAction(self.action_select_all)
        self.actions_menu.addAction(self.action_select_none)
        self.actions_menu.addAction(self.action_remove_empty)
        self.actions_menu.addSeparator()
        self.actions_menu.addAction(self.action_pca)

        self.tab1dlayout = QtGui.QVBoxLayout(self)
        self.tab1d_button_layout = QtGui.QHBoxLayout(self)
        self.tab1dlayout.addLayout(self.tab1d_button_layout)
        self.tab1d_button_layout.addWidget(self.subspace2d)
        self.tab1d_button_layout.addWidget(self.button_get_ranges)
        self.tab1d_button_layout.addWidget(self.button_store)
        self.tab1d_button_layout.addWidget(self.button_actions)

        self.filter_line_edit = QtGui.QLineEdit(self)
        self.filter_line_edit.setPlaceholderText("Enter space seperated search terms")
        self.filter_line_edit.textEdited.connect(functools.partial(self.onFilter, table=self.table1d))

        self.tab1dlayout.addWidget(self.filter_line_edit)
        self.tab1dlayout.addWidget(self.table1d)
        # self.tab1dlayout.addWidget(self.rankButton)
        # self.setCentralWidget(self.splitter)
        self.tab1d.setLayout(self.tab1dlayout)

        self.tabs.addTab(self.tab1d, "1d")

        self.resize(700, 500)

        if 0:
            for name in self.dataset.column_names:
                item = QtGui.QListWidgetItem(self.list1d)
                item.setText(name)
                item.setCheckState(False)
                # self.list1d.

        self.boxlayout = QtGui.QVBoxLayout(self)

        self.gridlayout = QtGui.QGridLayout()
        self.gridlayout.setColumnStretch(1, 1)
        self.gridlayout.setSpacing(0)
        self.gridlayout.setContentsMargins(2, 1, 2, 1)
        self.gridlayout.setAlignment(QtCore.Qt.AlignTop)
        row = 1

        self.selection_label = QtGui.QLabel("Use for computations:", self)
        self.gridlayout.addWidget(self.selection_label, row, 0)
        self.radio_button_all = QtGui.QRadioButton("Complete dataset", self)
        self.radio_button_selection = QtGui.QRadioButton("Selection", self)
        self.radio_button_all.setChecked(True)
        self.gridlayout.addWidget(self.radio_button_all, row, 1)
        row += 1
        self.gridlayout.addWidget(self.radio_button_selection, row, 1)
        row += 1

        def get():
            return str(self.grid_size)

        def set(value):
            self.grid_size = int(value)
        self.option_grid_size = Option(self, "grid size (for mutual info)", "32 64 128 256".split(), get, set)
        row = self.option_grid_size.add_to_grid_layout(row, self.gridlayout)
        # self.gridlayout.addWidget(self.option_grid_size.combobox); row += 1
        # self.gridlayout.addWidget(self.option_grid_size.combobox); row += 1

        self.boxlayout.addLayout(self.gridlayout)
        self.boxlayout.addWidget(self.tabs)
        # self.boxlayout.addWidget(self.rankButton)
        # self.setCentralWidget(self.splitter)
        self.setLayout(self.boxlayout)

        if "2" in options.get("open", ""):
            self.open(dim=2)
        if "3" in options.get("open", ""):
            self.open(dim=3)
        if "4" in options.get("open", ""):
            self.open(dim=4)

        self.fill_range_map()

    def open(self, dim=2):
        pairs1d = self.subspaceTables[1].getSelected()
        pairsprevd = self.subspaceTables[dim - 1].getSelected()
        # print pairs1d
        # print pairsprevd
        newpairs = list(joinpairs(pairs1d, pairsprevd))
        print(("newpairs", newpairs))
        if dim not in self.subspaceTables:
            self.tabNd = QtGui.QWidget(self.tabs)
            self.tableNd = SubspaceTable(self, self.tabNd, self.mainPanel, self.dataset, newpairs, dim, self.properties)
            self.tabNdlayout = QtGui.QVBoxLayout(self)
            self.tabNdButtonLayout = QtGui.QHBoxLayout(self)
            self.subspaceNd = QtGui.QPushButton("Create %dd subspaces" % (dim + 1), self.tab1d)
            self.plotNd = QtGui.QPushButton("Rank plot")
            self.exportNd = QtGui.QPushButton("Export ranking")
            if dim == len(self.dataset.column_names):
                self.subspaceNd.setDisabled(True)

            self.menu_calculate = QtGui.QMenu()
            self.button_calculate = QtGui.QToolButton()
            self.button_calculate.setText("Calculate")
            self.button_calculate.setPopupMode(QtGui.QToolButton.InstantPopup)
            # self.button_get_ranges.setToolButtonStyle(QtCore.Qt.ToolButtonTextUnderIcon)
            self.button_calculate.setMenu(self.menu_calculate)

            self.tabNdButtonLayout.addWidget(self.subspaceNd)
            self.tabNdButtonLayout.addWidget(self.button_calculate)
            # self.tabNdButtonLayout.addWidget(self.miNd)
            # self.tabNdButtonLayout.addWidget(self.correlationNd)
            self.tabNdButtonLayout.addWidget(self.exportNd)
            self.tabNdButtonLayout.addWidget(self.plotNd)
            self.tabNdlayout.addLayout(self.tabNdButtonLayout)
            self.subspaceNd.clicked.connect(functools.partial(self.open, dim=dim + 1))
            self.exportNd.clicked.connect(functools.partial(self.export, table=self.tableNd))
            self.plotNd.clicked.connect(functools.partial(self.rank_plot, table=self.tableNd))

            self.action_calculate_mi = QtGui.QAction("Calculate mutual information", self)
            self.action_calculate_correlation = QtGui.QAction("Calculate correlation", self)
            self.action_calculate_mi.triggered.connect(functools.partial(self.rankSubspaces, table=self.tableNd))
            self.action_calculate_correlation.triggered.connect(functools.partial(self.calculate_correlation, table=self.tableNd))
            self.menu_calculate.addAction(self.action_calculate_mi)
            self.menu_calculate.addAction(self.action_calculate_correlation)

            self.action_calculate_rank_correlation_MI_corr_kendall = QtGui.QAction("MI - correlation", self)
            self.action_calculate_rank_correlation_MI_abs_corr_kendall = QtGui.QAction("MI - abs(correlation)", self)

            self.action_calculate_rank_correlation_MI_corr_spearman = QtGui.QAction("MI - correlation", self)
            self.action_calculate_rank_correlation_MI_abs_corr_spearman = QtGui.QAction("MI - abs(correlation)", self)

            self.menu_correlation_kendall = self.menu_calculate.addMenu("Kendall's rank correlation")
            self.menu_correlation_spearman = self.menu_calculate.addMenu("Spearman's rank correlation")
            self.menu_correlation_kendall.setEnabled(False)
            self.menu_correlation_spearman.setEnabled(False)

            self.menu_correlation_kendall.addAction(self.action_calculate_rank_correlation_MI_corr_kendall)
            self.menu_correlation_kendall.addAction(self.action_calculate_rank_correlation_MI_abs_corr_kendall)

            self.menu_correlation_spearman.addAction(self.action_calculate_rank_correlation_MI_corr_spearman)
            self.menu_correlation_spearman.addAction(self.action_calculate_rank_correlation_MI_abs_corr_spearman)

            self.action_calculate_rank_correlation_MI_corr_kendall.triggered.connect(functools.partial(self.calculate_rank_correlation_kendall, table=self.tableNd))
            self.action_calculate_rank_correlation_MI_abs_corr_kendall.triggered.connect(functools.partial(self.calculate_rank_correlation_kendall, table=self.tableNd, absolute=True))

            self.action_calculate_rank_correlation_MI_corr_spearman.triggered.connect(functools.partial(self.calculate_rank_correlation_spearman, table=self.tableNd))
            self.action_calculate_rank_correlation_MI_abs_corr_spearman.triggered.connect(functools.partial(self.calculate_rank_correlation_spearman, table=self.tableNd, absolute=True))

            def func(index, name=""):
                print((name, index.row(), index.column()))
            self.tableNd.pressed.connect(functools.partial(func, name="pressed"))
            self.tableNd.entered.connect(functools.partial(func, name="entered"))
            self.tableNd.clicked.connect(functools.partial(func, name="clicked"))
            self.tableNd.activated.connect(functools.partial(func, name="activated"))

            def func(index, previous, name=""):
                print((name, index.row(), index.column(), previous.row(), previous.column()))
            self.selectionModel = self.tableNd.selectionModel()
            self.selectionModel.currentChanged.connect(functools.partial(func, name="currentChanged"))

            self.filter_Nd_line_edit = QtGui.QLineEdit(self)
            self.filter_Nd_line_edit.setPlaceholderText("Enter space seperated search terms")
            self.filter_Nd_line_edit.textEdited.connect(functools.partial(self.onFilter, table=self.tableNd))
            self.tabNdlayout.addWidget(self.filter_Nd_line_edit)

            self.tabNdlayout.addWidget(self.tableNd)
            # self.tab1dlayout.addWidget(self.rankButton)
            # self.setCentralWidget(self.splitter)
            self.tabNd.setLayout(self.tabNdlayout)
            self.subspaceTables[dim] = self.tableNd
            self.subspaceTabs[dim] = self.tabNd

            self.tabs.addTab(self.tabNd, "%dd" % dim)
            self.tabs.setCurrentWidget(self.tabNd)
        else:
            self.subspaceTables[dim].setPairs(newpairs)
            self.tabs.setCurrentWidget(self.subspaceTabs[dim])

    def onPca(self):
        # vaex.pca.
        pass

    def onFilter(self, text, table):
        table.set_filter_terms(text.split())

    def onStore(self):
        selected_pairs = self.table1d.getSelected()
        # error = False
        for pair in self.table1d.pairs:
            key = str(pair[0])
            # print repr(key+".use"), repr(pair in selected_pairs)
            self.properties[key + ".use"] = repr(pair in selected_pairs)
            if key in self.range_map:
                mi, ma = self.range_map[key]
                self.properties[key + ".min"] = repr(mi)
                self.properties[key + ".max"] = repr(ma)
            else:
                print(("min/max not present", key))
        print(("save to", self.properties_path))
        self.store_properties()
        dialog_info(self, "Stored", "Stored configuration to: %r" % self.properties_path)

    def load_properties(self):
        with open(self.properties_path, "rb") as f:
            self.properties = jprops.load_properties(f, collections.OrderedDict)

    def store_properties(self):
        with open(self.properties_path, "wb") as f:
            jprops.store_properties(f, self.properties)

    def fill_range_map(self):
        pairs = self.table1d.getSelected()
        for pair in pairs:
            mi, ma = self.table1d.get_range(pair)
            if mi is not None and ma is not None:
                self.range_map[pair[0]] = mi, ma

    def onSelectAll(self):
        pairs = self.table1d.pairs
        for pair in pairs:
            self.table1d.select(pair)

    def onSelectNone(self):
        pairs = self.table1d.pairs
        for pair in pairs:
            self.table1d.deselect(pair)

    def onRemoveEmpty(self):
        self.fill_range_map()
        pairs = self.table1d.getSelected()
        error = False
        for pair in pairs:
            print(pair)
            if pair[0] in self.range_map:
                min, max = self.range_map[pair[0]]
                if min == max:
                    print((pair, "empty", min, max))
                    self.table1d.deselect(pair)
            else:
                if not error:  # only give a warning once
                    dialog_error(self, "Min/max missing", "Min max missing for %s" % pair)
                    error = True

    def onCalculateMinMax(self):
        pairs = self.table1d.getSelected()
        logger.debug("estimate min/max for %r" % pairs)
        if self.dataset.is_local():
            executor = vaex.execution.Executor(buffer_size=buffer_size)
        else:
            executor = vaex.remote.ServerExecutor()

        expressions = [pair[0] for pair in pairs]
        assert len(pairs[0]) == 1
        self.range_map = {}
        try:
            with dialogs.ProgressExecution(self, "Calculating min/max", executor=executor) as progress:
                subspace = self.dataset.subspace(*expressions, executor=executor, delay=True)
                minmax = subspace.minmax()
                progress.add_task(minmax).end()
                progress.execute()
            ranges = minmax.get()
            self.table1d.setRanges(pairs, ranges)
            self.fill_range_map()
        except:
            logger.exception("Error in min/max or cancelled")
        # dialog.hide()

    def onCalculateMinMax3Sigma(self):
        pairs = self.table1d.getSelected()

        expressions = [pair[0] for pair in pairs]
        if self.dataset.is_local():
            executor = vaex.execution.Executor(buffer_size=buffer_size)
        else:
            executor = vaex.remote.ServerExecutor()

        if self.dataset.is_local():
            executor = vaex.execution.Executor()
        else:
            executor = vaex.remote.ServerExecutor()
        subspace = self.dataset.subspace(*expressions, executor=executor, delay=True)
        means = subspace.mean()
        with dialogs.ProgressExecution(self, "Calculating mean", executor=executor) as progress:
            progress.add_task(means).end()
            progress.execute()
        logger.debug("get means")
        means = means.get()
        logger.debug("got means")

        vars = subspace.var(means=means)
        with dialogs.ProgressExecution(self, "Calculating variance", executor=executor) as progress:
            progress.add_task(vars).end()
            progress.execute()
        # limits  = limits.get()
        vars = vars.get()
        stds = vars**0.5
        sigmas = 3
        ranges = list(zip(means - sigmas * stds, means + sigmas * stds))
        self.table1d.setRanges(pairs, ranges)
        self.fill_range_map()

    def calculate_rank_correlation_kendall(self, table, absolute=False):
        print(("kendall", table, absolute))
        pairs = table.getSelected()
        mi_values = [table.qualities[pair] for pair in pairs]
        correlation_values = [table.correlation_map[pair] for pair in pairs]
        ranking_mi = np.argsort(mi_values)
        if absolute:
            ranking_correlation = np.argsort(np.abs(correlation_values))
        else:
            ranking_correlation = np.argsort(correlation_values)
        N = len(pairs)
        A = np.zeros((N, N))
        B = np.zeros((N, N))
        for i in range(N):
            for j in range(N):
                A[i, j] = np.sign(ranking_mi[i] - ranking_mi[j])
                B[i, j] = np.sign(ranking_correlation[i] - ranking_correlation[j])
        AB = 0
        AA = 0
        BB = 0
        for i in range(N):
            for j in range(N):
                AB += A[i, j] * B[i, j]
                AA += A[i, j]**2
                BB += B[i, j]**2

    def calculate_rank_correlation_spearman(self, table, absolute=False):
        print(("spearman", table, absolute))

    def calculate_correlation(self, table):
        print(("calculate correlation for ", table))
        pairs = table.getSelected()

        expressions = set()
        for pair in pairs:
            for expression in pair:
                expressions.add(expression)
        expressions = list(expressions)
        if self.dataset.is_local():
            executor = vaex.execution.Executor(buffer_size=buffer_size)
        else:
            executor = vaex.remote.ServerExecutor()

        def on_error(exc):
            raise exc
        if 1:
            # subspace = self.dataset(*expressions, executor=executor, delay=True)
            subspaces = self.dataset.subspaces(pairs, executor=executor, delay=True)
            means_promise = subspaces.mean()
            # print means_promise, type(means_promise), subs
            with dialogs.ProgressExecution(self, "Calculating means", executor=executor) as progress:
                progress.add_task(means_promise)
                progress.execute()
            means = means_promise.get()

            variances_promise = subspaces.var(means=means)
            with dialogs.ProgressExecution(self, "Calculating variances", executor=executor) as progress:
                progress.add_task(variances_promise)
                progress.execute()
            vars = variances_promise.get()

            # means = subspaces._unpack(means_packed)
            # vars = subspaces._unpack(vars_packed)
            tasks = []
            with dialogs.ProgressExecution(self, "Calculating correlation", executor=executor) as progress:
                for subspace, mean, var in zip(subspaces.subspaces, means, vars):
                    task = subspace.correlation(means=mean, vars=var)
                    progress.add_task(task).end()
                    tasks.append(task)
                progress.execute()
            correlations = [task.get() for task in tasks]

            correlation_map = dict(zip(pairs, correlations))
            table.set_correlations(correlation_map)
            return
            # mean_map = dict(zip(expressions, means))
            # var_map = dict(zip(expressions, variances))
        else:
            mean_map = {}

            def on_error(exc):
                raise exc
            for expression in expressions:
                subspace = self.dataset(expression, executor=executor, delay=True)

                def assign(mean_list, expression=expression):
                    logger.debug("assigning %r to %s", mean_list, expression)
                    mean_map[expression] = mean_list
                subspace.mean().then(assign, on_error).end()
            with dialogs.ProgressExecution(self, "Calculating means", executor=executor):
                executor.execute()

            var_map = {}
            for expression in expressions:
                subspace = self.dataset(expression, executor=executor, delay=True)

                def assign(mean_list, expression=expression):
                    logger.debug("assigning %r to %s", mean_list, expression)
                    var_map[expression] = mean_list[0].tolist()
                subspace.var(means=mean_map[expression]).then(assign, on_error).end()
            with dialogs.ProgressExecution(self, "Calculating variances", executor=executor):
                executor.execute()

            means = [mean_map[expressions[0]] for expressions in pairs]
            variances = [var_map[expressions[0]] for expressions in pairs]

        correlation_map = {}
        for pair in pairs:
            means = [mean_map[expression] for expression in pair]
            vars = [var_map[expression] for expression in pair]
            subspace = self.dataset(*pair, executor=executor, delay=True)

            def assign(correlation, pair=pair):
                logger.debug("assigning %r to %s", correlation, pair)
                correlation_map[pair] = correlation
            subspace.correlation(means, vars).then(assign, on_error).end()

        with dialogs.ProgressExecution(self, "Calculating correlation", executor=executor):
            executor.execute()

        table.set_correlations(correlation_map)
        return

        jobsManager = vaex.dataset.JobsManager()
        expressions = set()
        for pair in pairs:
            for expression in pair:
                expressions.add(expression)
        expressions = list(expressions)
        print("means")
        with ProgressExecution(self, "Calculating means") as progress:
            means = jobsManager.calculate_mean(self.dataset, use_mask=self.radio_button_selection.isChecked(), expressions=expressions, feedback=progress.progress)
        mean_map = dict(list(zip(expressions, means)))
        centered_expressions_map = {expression: "(%s - %.20e)" % (expression, mean) for (expression, mean) in list(mean_map.items())}
        variances_expressions_map = {expression: "%s**2" % centered_expressions for expression, centered_expressions in list(centered_expressions_map.items())}
        with ProgressExecution(self, "Calculating variances") as progress:
            variances = jobsManager.calculate_mean(self.dataset, use_mask=self.radio_button_selection.isChecked(), expressions=list(variances_expressions_map.values()), feedback=progress.progress)
        variances_map = dict(list(zip(list(variances_expressions_map.keys()), variances)))

        covariances_expressions = []
        for pair in pairs:
            centered_expressions = [centered_expressions_map[expression] for expression in pair]
            covariance_expression = "*".join(centered_expressions)
            covariances_expressions.append(covariance_expression)

        print(covariances_expressions)
        with ProgressExecution(self, "Calculating covariances") as progress:
            # progress.progress(20)
            covariances = jobsManager.calculate_mean(self.dataset, use_mask=self.radio_button_selection.isChecked(), expressions=covariances_expressions, feedback=progress.progress)
            # progress.progress(20)
        print(variances)
        print(covariances)

        correlation_map = {}
        for pair, covariance in zip(pairs, covariances):
            normalization = 1
            for expression in pair:
                normalization *= np.sqrt(variances_map[expression])
            correlation_map[pair] = covariance / normalization
        table.set_correlations(correlation_map)

        return

    def export(self, table):
        print(("export", table))
        basename, ext = os.path.splitext(self.dataset.path)
        path = basename + ("-ranking-%dd" % table.dim) + ".properties"
        filename = get_path_save(self, path=path, title="Export ranking", file_mask="properties file *.properties")
        expressions = set()
        if filename:
            counts = 0
            with open(filename, "w") as file:
                for pair in table.pairs:
                    if pair in table.qualities:
                        file.write("%s.mutual_information=%f\n" % (".".join(pair), table.qualities[pair]))
                        counts += 1
                    if pair in table.correlation_map:
                        file.write("%s.correlation_coefficient=%f\n" % (".".join(pair), table.correlation_map[pair]))
                        counts += 1
                    expressions.update(pair)
                for expression in expressions:
                    mi, ma = self.range_map[expression]
                    file.write("%s.min=%f\n" % (expression, mi))
                    file.write("%s.max=%f\n" % (expression, ma))
                dialog_info(self, "Wrote ranking file", "wrote %d lines" % counts)

    def rankSubspaces(self, table):
        self.fill_range_map()

        pairs = table.getSelected()
        error = False
        ranges = []

        for pair in pairs:
            for expression in pair:
                if expression not in self.range_map:
                    error = True
                    print(("missing", expression))
        if error:
            dialog_error(self, "Missing min/max", "Please calculate the minimum and maximum for the dimensions")
            return

        # expressions = [pair[0] for pair in pairs]
        # executor = vaex.execution.Executor(buffer_size=buffer_size)
        if self.dataset.is_local():
            executor = vaex.execution.Executor(buffer_size=buffer_size)
        else:
            executor = vaex.remote.ServerExecutor()

        tasks = []
        with dialogs.ProgressExecution(self, "Calculating mutual information", executor=executor) as progress:
            for pair in pairs:
                limits = [self.range_map[expr] for expr in pair]
                task = self.dataset(*pair, executor=executor, delay=True).mutual_information(limits=limits, size=self.grid_size)
                progress.add_task(task).end()
                tasks.append(task)
            if not progress.execute():
                return
        logger.debug("get means")
        mutual_information = [task.get() for task in tasks]

        # mutual_information_list = [MI_map[pair] for pair in pairs]
        table.setQualities(pairs, mutual_information)
        return

        print(table)
        qualities = []
        pairs = table.getSelected()

        if 0:
            for pair in pairs:
                dim = len(pair)
                # if dim == 2:
                columns = [self.dataset.columns[name] for name in pair]
                print(pair)
                information = vaex.kld.kld_shuffled(columns, mask=mask)
                qualities.append(information)
                # print pair
        if 0:
            dialog = QtGui.QProgressDialog("Calculating Mutual information", "Abort", 0, 1000, self)
            dialog.show()

            def feedback(percentage):
                print(percentage)
                dialog.setValue(int(percentage * 10))
                QtCore.QCoreApplication.instance().processEvents()
                if dialog.wasCanceled():
                    return True
        with ProgressExecution(self, "Calculating Mutual information") as progress:
            qualities = vaex.kld.kld_shuffled_grouped(self.dataset, self.range_map, pairs, feedback=progress.progress, use_mask=self.radio_button_selection.isChecked())
            # dialog.hide()
        if qualities is not None:
            print(qualities)
            table.setQualities(pairs, qualities)

    def rank_plot(self, table):
        plot = RankPlot(self, table)
        plot.show()


__author__ = 'maartenbreddels'
import os
import numpy as np
import logging
import vaex
import vaex.utils
import vaex.execution
import vaex.export
import vaex.hdf5.dataset

max_length = int(1e5)

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
try:
    import h5py
except:
    if not on_rtd:
        raise

logger = logging.getLogger("vaex.hdf5.export")

def export_hdf5_v1(dataset, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=True):
    """
    :param DatasetLocal dataset: dataset to export
    :param str path: path for file
    :param lis[str] column_names: list of column names to export or None for all columns
    :param str byteorder: = for native, < for little endian and > for big endian
    :param bool shuffle: export rows in random order
    :param bool selection: export selection or not
    :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
        or a default progress bar when progress=True
    :param: bool virtual: When True, export virtual columns
    :return:
    """

    if selection:
        if selection == True:  # easier to work with the name
            selection = "default"
    # first open file using h5py api
    with h5py.File(path, "w") as h5file_output:

        h5data_output = h5file_output.require_group("data")
        # i1, i2 = dataset.current_slice
        N = len(dataset) if not selection else dataset.selected_length(selection)
        if N == 0:
            raise ValueError("Cannot export empty table")
        logger.debug("virtual=%r", virtual)
        logger.debug("exporting %d rows to file %s" % (N, path))
        # column_names = column_names or (dataset.get_column_names() + (list(dataset.virtual_columns.keys()) if virtual else []))
        column_names = column_names or dataset.get_column_names(virtual=virtual, strings=True)

        logger.debug("exporting columns(hdf5): %r" % column_names)
        for column_name in column_names:
            if column_name in dataset.get_column_names(strings=True):
                column = dataset.columns[column_name]
                shape = (N,) + column.shape[1:]
                dtype = column.dtype
            else:
                dtype = np.float64().dtype
                shape = (N,)
            if dtype.type == np.datetime64:
                array = h5file_output.require_dataset("/data/%s" % column_name, shape=shape, dtype=np.int64)
                array.attrs["dtype"] = dtype.name
            else:
                try:
                    array = h5file_output.require_dataset("/data/%s" % column_name, shape=shape, dtype=dtype.newbyteorder(byteorder))
                except:
                    logging.exception("error creating dataset for %r, with type %r " % (column_name, dtype))
            array[0] = array[0]  # make sure the array really exists
        random_index_name = None
        column_order = list(column_names)  # copy
        if shuffle:
            random_index_name = "random_index"
            while random_index_name in dataset.get_column_names():
                random_index_name += "_new"
            shuffle_array = h5file_output.require_dataset("/data/" + random_index_name, shape=(N,), dtype=byteorder + "i8")
            shuffle_array[0] = shuffle_array[0]
            column_order.append(random_index_name)  # last item
        h5data_output.attrs["column_order"] = ",".join(column_order)  # keep track or the ordering of columns

    # after this the file is closed,, and reopen it using out class
    dataset_output = vaex.hdf5.dataset.Hdf5MemoryMapped(path, write=True)

    column_names = vaex.export._export(dataset_input=dataset,
                                       dataset_output=dataset_output,
                                       path=path,
                                       random_index_column=random_index_name,
                                       column_names=column_names,
                                       selection=selection,
                                       shuffle=shuffle,
                                       byteorder=byteorder,
                                       progress=progress)
    import getpass
    import datetime
    user = getpass.getuser()
    date = str(datetime.datetime.now())
    source = dataset.path
    description = "file exported by vaex, by user %s, on date %s, from source %s" % (user, date, source)
    if dataset.description:
        description += "previous description:\n" + dataset.description
    dataset_output.copy_metadata(dataset)
    dataset_output.description = description
    logger.debug("writing meta information")
    dataset_output.write_meta()
    dataset_output.close_files()
    return

def export_hdf5(dataset, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):
    """
    :param DatasetLocal dataset: dataset to export
    :param str path: path for file
    :param lis[str] column_names: list of column names to export or None for all columns
    :param str byteorder: = for native, < for little endian and > for big endian
    :param bool shuffle: export rows in random order
    :param bool selection: export selection or not
    :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
        or a default progress bar when progress=True
    :param: bool virtual: When True, export virtual columns
    :return:
    """

    if selection:
        if selection == True:  # easier to work with the name
            selection = "default"
    # first open file using h5py api
    with h5py.File(path, "w") as h5file_output:

        h5table_output = h5file_output.require_group("/table")
        h5table_output.attrs["type"] = "table"
        h5columns_output = h5file_output.require_group("/table/columns")
        # i1, i2 = dataset.current_slice
        N = len(dataset) if not selection else dataset.selected_length(selection)
        if N == 0:
            raise ValueError("Cannot export empty table")
        logger.debug("virtual=%r", virtual)
        logger.debug("exporting %d rows to file %s" % (N, path))
        # column_names = column_names or (dataset.get_column_names() + (list(dataset.virtual_columns.keys()) if virtual else []))
        column_names = column_names or dataset.get_column_names(virtual=virtual, strings=True)

        logger.debug("exporting columns(hdf5): %r" % column_names)
        for column_name in column_names:
            if column_name in dataset.get_column_names(strings=True):
                column = dataset.columns[column_name]
                shape = (N,) + column.shape[1:]
                dtype = column.dtype
            else:
                dtype = np.float64().dtype
                shape = (N,)
            h5column_output = h5columns_output.require_group(column_name)
            if dtype.type == np.datetime64:
                array = h5column_output.require_dataset('data', shape=shape, dtype=np.int64)
                array.attrs["dtype"] = dtype.name
            else:
                try:
                    array = h5column_output.require_dataset('data', shape=shape, dtype=dtype.newbyteorder(byteorder))
                except:
                    logging.exception("error creating dataset for %r, with type %r " % (column_name, dtype))
            array[0] = array[0]  # make sure the array really exists

            data = dataset.evaluate(column_name, 0, 1)
            if np.ma.isMaskedArray(data):
                mask = h5column_output.require_dataset('mask', shape=shape, dtype=np.bool)
                mask[0] = mask[0]  # make sure the array really exists
        random_index_name = None
        column_order = list(column_names)  # copy
        if shuffle:
            random_index_name = "random_index"
            while random_index_name in dataset.get_column_names():
                random_index_name += "_new"
            shuffle_array = h5columns_output.require_dataset(random_index_name + "/data", shape=(N,), dtype=byteorder + "i8")
            shuffle_array[0] = shuffle_array[0]
            column_order.append(random_index_name)  # last item
        h5columns_output.attrs["column_order"] = ",".join(column_order)  # keep track or the ordering of columns

    # after this the file is closed,, and reopen it using out class
    dataset_output = vaex.hdf5.dataset.Hdf5MemoryMapped(path, write=True)

    column_names = vaex.export._export(dataset_input=dataset,
                                       dataset_output=dataset_output,
                                       path=path,
                                       random_index_column=random_index_name,
                                       column_names=column_names,
                                       selection=selection,
                                       shuffle=shuffle,
                                       byteorder=byteorder,
                                       progress=progress,
                                       sort=sort,
                                       ascending=ascending)
    import getpass
    import datetime
    user = getpass.getuser()
    date = str(datetime.datetime.now())
    source = dataset.path
    description = "file exported by vaex, by user %s, on date %s, from source %s" % (user, date, source)
    if dataset.description:
        description += "previous description:\n" + dataset.description
    dataset_output.copy_metadata(dataset)
    dataset_output.description = description
    logger.debug("writing meta information")
    dataset_output.write_meta()
    dataset_output.close_files()
    return


# -*- coding: utf-8 -*-

import struct

def getinfo(filename, seek=None):
	"""Read header data from Gadget data file 'filename' with Gadget file
	type 'gtype'. Returns offsets of positions and velocities."""
	DESC = '=I4sII'                                # struct formatting string
	HEAD = '=I6I6dddii6iiiddddii6ii60xI'        # struct formatting string
	keys = ('Npart', 'Massarr', 'Time', 'Redshift', 'FlagSfr', 'FlagFeedback', 'Nall', 'FlagCooling', 'NumFiles', 'BoxSize', 'Omega0', 'OmegaLambda', 'HubbleParam', 'FlagAge', 'FlagMetals', 'NallHW', 'flag_entr_ics', 'filename')
	f = open(filename, 'rb')

	"""Detects Gadget file type (type 1 or 2; resp. without or with the 16
	byte block headers)."""
	firstbytes = struct.unpack('I',f.read(4))
	if firstbytes[0] == 8:
		gtype = 2
	else:
		gtype = 1
	if gtype == 2:
		f.seek(16)
	else:
	 	f.seek(0)
	if seek is not None:
		f.seek(seek)
	raw = struct.unpack(HEAD,f.read(264))[1:-1]
	values = (raw[:6], raw[6:12]) + raw[12:16] + (raw[16:22],) + raw[22:30] + (raw[30:36], raw[36], filename)
	header = dict(list(zip(keys, values)))
	f.close()

	if gtype == 2:
		posoffset = (2*16 + (8 + 256))
	else:
		posoffset = (8 + 256)

	Npart = sum(header['Npart'])
	if gtype == 2:
		veloffset = 3*16 + (8 + 256) + (8 + 3*4*Npart)
	else:
		veloffset= (8 + 256) + (8 + 3*4*Npart)
	return Npart, posoffset+4, veloffset+4, header


__author__ = 'maartenbreddels'
import os
import sys
import collections
import numpy as np
import logging
import concurrent.futures
import threading

import vaex
import vaex.utils
import vaex.execution
import vaex.file.colfits
import vaex.file.other
from vaex.column import ColumnStringArrow, str_type, _to_string_sequence

max_length = int(1e5)

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
try:
    import h5py
except:
    if not on_rtd:
        raise
# from vaex.dataset import DatasetLocal

logger = logging.getLogger("vaex.export")
progress_lock = threading.Lock()

class ProgressStatus(object):
    pass

def _export(dataset_input, dataset_output, random_index_column, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):
    """
    :param DatasetLocal dataset: dataset to export
    :param str path: path for file
    :param lis[str] column_names: list of column names to export or None for all columns
    :param str byteorder: = for native, < for little endian and > for big endian
    :param bool shuffle: export rows in random order
    :param bool selection: export selection or not
    :param progress: progress callback that gets a progress fraction as argument and should return True to continue
    :return:
    """

    if selection:
        if selection == True:  # easier to work with the name
            selection = "default"

    N = len(dataset_input) if not selection else dataset_input.selected_length(selection)
    if N == 0:
        raise ValueError("Cannot export empty table")

    if shuffle and sort:
        raise ValueError("Cannot shuffle and sort at the same time")

    if shuffle:
        shuffle_array = dataset_output.columns[random_index_column]

    partial_shuffle = shuffle and len(dataset_input) != N

    order_array = None
    order_array_inverse = None

    # for strings we also need the inverse order_array, keep track of that
    has_strings = any([dataset_input.dtype(k) == str_type for k in column_names])

    if partial_shuffle:
        # if we only export a portion, we need to create the full length random_index array, and
        shuffle_array_full = np.random.choice(len(dataset_input), len(dataset_input), replace=False)
        # then take a section of it
        shuffle_array[:] = shuffle_array_full[shuffle_array_full < N]
        del shuffle_array_full
        order_array = shuffle_array
    elif shuffle:
        # better to do this in memory
        shuffle_array_memory = np.random.choice(N, N, replace=False)
        shuffle_array[:] = shuffle_array_memory
        order_array = shuffle_array
    if order_array is not None:
        indices_r = np.zeros_like(order_array)
        indices_r[order_array] = np.arange(len(order_array))
        order_array_inverse = indices_r
        del indices_r

    if sort:
        if selection:
            raise ValueError("sorting selections not yet supported")
        # these indices sort the input array, but we evaluate the input in sequential order and write it out in sorted order
        # e.g., not b[:] = a[indices]
        # but b[indices_r] = a
        logger.info("sorting...")
        indices = np.argsort(dataset_input.evaluate(sort))
        indices_r = np.zeros_like(indices)
        indices_r[indices] = np.arange(len(indices))
        if has_strings:
            # in this case we already have the inverse ready
            order_array_inverse = indices if ascending else indices[:--1]
        else:
            del indices
        order_array = indices_r if ascending else indices_r[::-1]
        logger.info("sorting done")

    if progress == True:
        progress = vaex.utils.progressbar_callable(title="exporting")
    progress = progress or (lambda value: True)
    progress_total = len(column_names) * len(dataset_input)
    progress_status = ProgressStatus()
    progress_status.cancelled = False
    progress_status.value = 0
    if selection:
        full_mask = dataset_input.evaluate_selection_mask(selection)
    else:
        full_mask = None

    sparse_groups = collections.defaultdict(list)
    sparse_matrices = {}  # alternative to a set of matrices, since they are not hashable
    string_columns = []
    futures = []

    thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
    if True:
        for column_name in column_names:
            sparse_matrix = dataset_output._sparse_matrix(column_name)
            if sparse_matrix is not None:
                # sparse columns are written differently
                sparse_groups[id(sparse_matrix)].append(column_name)
                sparse_matrices[id(sparse_matrix)] = sparse_matrix
                continue
            logger.debug("  exporting column: %s " % column_name)
            future = thread_pool.submit(_export_column, dataset_input, dataset_output, column_name, full_mask,
                shuffle, sort, selection, N, order_array, order_array_inverse, progress_status)
            futures.append(future)

    done = False
    while not done:
        done = True
        for future in futures:
            try:
                future.result(0.1/4)
            except concurrent.futures.TimeoutError:
                done = False
                break
        if not done:
            if not progress(progress_status.value / float(progress_total)):
                progress_status.cancelled = True

    for sparse_matrix_id, column_names in sparse_groups.items():
        sparse_matrix = sparse_matrices[sparse_matrix_id]
        for column_name in column_names:
            assert not shuffle
            assert selection in [None, False]
            column = dataset_output.columns[column_name]
            column.matrix.data[:] = dataset_input.columns[column_name].matrix.data
            column.matrix.indptr[:] = dataset_input.columns[column_name].matrix.indptr
            column.matrix.indices[:] = dataset_input.columns[column_name].matrix.indices
    return column_names

def _export_column(dataset_input, dataset_output, column_name, full_mask, shuffle, sort, selection, N,
    order_array, order_array_inverse, progress_status):

        if 1:
            block_scope = dataset_input._block_scope(0, vaex.execution.buffer_size_default)
            to_array = dataset_output.columns[column_name]
            dtype = dataset_input.dtype(column_name)
            if shuffle or sort:  # we need to create a in memory copy, otherwise we will do random writes which is VERY inefficient
                to_array_disk = to_array
                if np.ma.isMaskedArray(to_array):
                    to_array = np.empty_like(to_array_disk)
                else:
                    if dtype == str_type:
                        # we create an empty column copy
                        to_array = to_array._zeros_like()
                    else:
                        to_array = np.zeros_like(to_array_disk)
            to_offset = 0  # we need this for selections
            count = len(dataset_input) if not selection else dataset_input.length_unfiltered()
            is_string = dtype == str_type
            # TODO: if no filter, selection or mask, we can choose the quick path for str
            string_byte_offset = 0

            for i1, i2 in vaex.utils.subdivide(count, max_length=max_length):
                logger.debug("from %d to %d (total length: %d, output length: %d)", i1, i2, len(dataset_input), N)
                block_scope.move(i1, i2)
                if selection:
                    mask = full_mask[i1:i2]
                    values = dataset_input.evaluate(column_name, i1=i1, i2=i2, filtered=False) #selection=selection)
                    values = values[mask]
                    no_values = len(values)
                    if no_values:
                        if is_string:
                            to_column = to_array
                            assert isinstance(to_column, ColumnStringArrow)
                            from_sequence = _to_string_sequence(values)
                            to_sequence = to_column.string_sequence.slice(to_offset, to_offset+no_values, string_byte_offset)
                            string_byte_offset += to_sequence.fill_from(from_sequence)
                            to_offset += no_values
                        else:
                            fill_value = np.nan if dtype.kind == "f" else None
                            # assert np.ma.isMaskedArray(to_array) == np.ma.isMaskedArray(values), "to (%s) and from (%s) array are not of both masked or unmasked (%s)" %\
                            # (np.ma.isMaskedArray(to_array), np.ma.isMaskedArray(values), column_name)
                            if dtype.type == np.datetime64:
                                values = values.view(np.int64)
                            if np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(values):
                                to_array.data[to_offset:to_offset + no_values] = values.filled(fill_value)
                                to_array.mask[to_offset:to_offset + no_values] = values.mask
                            elif not np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(values):
                                to_array[to_offset:to_offset + no_values] = values.filled(fill_value)
                            else:
                                to_array[to_offset:to_offset + no_values] = values
                            to_offset += no_values
                else:
                    values = dataset_input.evaluate(column_name, i1=i1, i2=i2)
                    if is_string:
                        no_values = len(values)
                        # for strings, we don't take sorting/shuffling into account when building the structure
                        to_column = to_array
                        assert isinstance(to_column, ColumnStringArrow)
                        from_sequence = _to_string_sequence(values)
                        to_sequence = to_column.string_sequence.slice(i1, i2, string_byte_offset)
                        string_byte_offset += to_sequence.fill_from(from_sequence)
                    else:
                        assert len(values) == (i2-i1)
                        fill_value = np.nan if dtype.kind == "f" else None
                        # assert np.ma.isMaskedArray(to_array) == np.ma.isMaskedArray(values), "to (%s) and from (%s) array are not of both masked or unmasked (%s)" %\
                        # (np.ma.isMaskedArray(to_array), np.ma.isMaskedArray(values), column_name)
                        if dtype.type == np.datetime64:
                            values = values.view(np.int64)
                        if shuffle or sort:
                            indices = order_array[i1:i2]
                            if np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(values):
                                to_array.data[indices] = values.filled(fill_value)
                                to_array.mask[indices] = values.mask
                            elif not np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(values):
                                to_array[indices] = values.filled(fill_value)
                            else:
                                to_array[indices] = values
                        else:
                            if np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(values):
                                to_array.data[i1:i2] = values.filled(fill_value)
                                to_array.mask[i1:i2] = values.mask
                            elif np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(values):
                                to_array[i1:i2] = values.filled(fill_value)
                            else:
                                to_array[i1:i2] = values
                with progress_lock:
                    progress_status.value += i2 - i1
                if progress_status.cancelled:
                    break
                #if not progress(progress_value / float(progress_total)):
                #    break
            if is_string:  # write out the last index
                to_column = to_array
                if selection:
                    to_column.indices[to_offset] = string_byte_offset
                else:
                    to_column.indices[count] = string_byte_offset
            if shuffle or sort:  # write to disk in one go
                if dtype == str_type:  # strings are sorted afterwards
                    view = to_array.string_sequence.lazy_index(order_array_inverse)
                    to_array_disk.string_sequence.fill_from(view)
                else:
                    if np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(to_array_disk):
                        to_array_disk.data[:] = to_array.data
                        to_array_disk.mask[:] = to_array.mask
                    else:
                        to_array_disk[:] = to_array

def export_fits(dataset, path, column_names=None, shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):
    """
    :param DatasetLocal dataset: dataset to export
    :param str path: path for file
    :param lis[str] column_names: list of column names to export or None for all columns
    :param bool shuffle: export rows in random order
    :param bool selection: export selection or not
    :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
            or a default progress bar when progress=True
    :param: bool virtual: When True, export virtual columns
    :return:
    """
    if shuffle:
        random_index_name = "random_index"
        while random_index_name in dataset.get_column_names():
            random_index_name += "_new"

    column_names = column_names or dataset.get_column_names(virtual=virtual, strings=True)
    logger.debug("exporting columns(fits): %r" % column_names)
    N = len(dataset) if not selection else dataset.selected_length(selection)
    data_types = []
    data_shapes = []
    ucds = []
    units = []
    for column_name in column_names:
        if column_name in dataset.get_column_names(strings=True):
            column = dataset.columns[column_name]
            shape = (N,) + column.shape[1:]
            dtype = column.dtype
            if dataset.dtype(column_name) == str_type:
                max_length = dataset[column_name].apply(lambda x: len(x)).max(selection=selection)
                dtype = np.dtype('S'+str(int(max_length)))
        else:
            dtype = np.float64().dtype
            shape = (N,)
        ucds.append(dataset.ucds.get(column_name))
        units.append(dataset.units.get(column_name))
        data_types.append(dtype)
        data_shapes.append(shape)

    if shuffle:
        column_names.append(random_index_name)
        data_types.append(np.int64().dtype)
        data_shapes.append((N,))
        ucds.append(None)
        units.append(None)
    else:
        random_index_name = None

    # TODO: all expressions can have missing values.. how to support that?
    null_values = {key: dataset.columns[key].fill_value for key in dataset.get_column_names() if dataset.is_masked(key) and dataset.dtype(key).kind != "f"}
    vaex.file.colfits.empty(path, N, column_names, data_types, data_shapes, ucds, units, null_values=null_values)
    if shuffle:
        del column_names[-1]
        del data_types[-1]
        del data_shapes[-1]
    dataset_output = vaex.file.other.FitsBinTable(path, write=True)
    _export(dataset_input=dataset, dataset_output=dataset_output, path=path, random_index_column=random_index_name,
            column_names=column_names, selection=selection, shuffle=shuffle,
            progress=progress, sort=sort, ascending=ascending)
    dataset_output.close_files()

def export_hdf5_v1(dataset, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=True):
    kwargs = locals()
    import vaex.hdf5.export
    vaex.hdf5.export.export_hdf5_v1(**kwargs)

def export_hdf5(dataset, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):
    kwargs = locals()
    import vaex.hdf5.export
    vaex.hdf5.export.export_hdf5(**kwargs)

if __name__ == "__main__":
    sys.exit(main(sys.argv))

def main(argv):
    import argparse
    parser = argparse.ArgumentParser(argv[0])
    parser.add_argument('--verbose', '-v', action='count', default=0)
    parser.add_argument('--quiet', '-q', default=False, action='store_true', help="do not output anything")
    parser.add_argument('--list', '-l', default=False, action='store_true', help="list columns of input")
    parser.add_argument('--progress', help="show progress (default: %(default)s)", default=True, action='store_true')
    parser.add_argument('--no-progress', dest="progress", action='store_false')
    parser.add_argument('--shuffle', "-s", dest="shuffle", action='store_true', default=False)
    parser.add_argument('--sort', dest="sort", default=None)
    parser.add_argument('--virtual', dest="virtual", action='store_true', default=False, help="Also export virtual columns")
    parser.add_argument('--fraction', "-f", dest="fraction", type=float, default=1.0, help="fraction of input dataset to export")
    parser.add_argument('--filter', dest="filter", default=None, help="filter to apply before exporting")

    subparsers = parser.add_subparsers(help='type of input source', dest="task")

    parser_soneira = subparsers.add_parser('soneira', help='create soneira peebles dataset')
    parser_soneira.add_argument('output', help='output file')
    parser_soneira.add_argument("columns", help="list of columns to export (or all when empty)", nargs="*")
    parser_soneira.add_argument('--dimension', '-d', type=int, help='dimensions', default=4)
    # parser_soneira.add_argument('--eta','-e', type=int, help='dimensions', default=3)
    parser_soneira.add_argument('--max-level', '-m', type=int, help='dimensions', default=28)
    parser_soneira.add_argument('--lambdas', '-l', type=int, help='lambda values for fractal', default=[1.1, 1.3, 1.6, 2.])

    parser_tap = subparsers.add_parser('tap', help='use TAP (Table Access Protocol) as source')
    parser_tap.add_argument("tap_url", help="input source or file")
    parser_tap.add_argument("table_name", help="input source or file")
    parser_tap.add_argument("output", help="output file (ends in .fits or .hdf5)")
    parser_tap.add_argument("columns", help="list of columns to export (or all when empty)", nargs="*")

    parser_file = subparsers.add_parser('file', help='use a file as source (e.g. .hdf5, .fits, .vot (VO table), .asc (ascii)')
    parser_file.add_argument("input", help="input source or file, when prefixed with @ it is assumed to be a text file with a file list (one file per line)")
    parser_file.add_argument("output", help="output file (ends in .fits or .hdf5)")
    parser_file.add_argument("columns", help="list of columns to export (or all when empty)", nargs="*")

    parser_file = subparsers.add_parser('csv', help='use a csv file as source (e.g. .hdf5, .fits, .vot (VO table), .asc (ascii)')
    parser_file.add_argument("input", help="input source or file, when prefixed with @ it is assumed to be a text file with a file list (one file per line)")
    parser_file.add_argument("output", help="output file (ends in .hdf5)")
    parser_file.add_argument("columns", help="list of columns to export (or all when empty)", nargs="*")

    args = parser.parse_args(argv[1:])

    verbosity = ["ERROR", "WARNING", "INFO", "DEBUG"]
    logging.getLogger("vaex").setLevel(verbosity[min(3, args.verbose)])
    dataset = None
    if args.task == "soneira":
        if vaex.utils.check_memory_usage(4 * 8 * 2**args.max_level, vaex.utils.confirm_on_console):
            if not args.quiet:
                print("generating soneira peebles dataset...")
            dataset = vaex.file.other.SoneiraPeebles(args.dimension, 2, args.max_level, args.lambdas)
        else:
            return 1
    if args.task == "tap":
        dataset = vaex.dataset.DatasetTap(args.tap_url, args.table_name)
        if not args.quiet:
            print("exporting from {tap_url} table name {table_name} to {output}".format(tap_url=args.tap_url, table_name=args.table_name, output=args.output))
    if args.task == "csv":
        # dataset = vaex.dataset.DatasetTap(args.tap_url, args.table_name)
        if not args.quiet:
            print("exporting from {input} to {output}".format(input=args.input, output=args.output))
    if args.task == "file":
        if args.input[0] == "@":
            inputs = open(args.input[1:]).readlines()
            dataset = vaex.open_many(inputs)
        else:
            dataset = vaex.open(args.input)
        if not args.quiet:
            print("exporting from {input} to {output}".format(input=args.input, output=args.output))

    if dataset is None and args.task not in ["csv"]:
        if not args.quiet:
            print("Cannot open input")
        return 1
    if dataset:
        dataset.set_active_fraction(args.fraction)
    if args.list:
        if not args.quiet:
            print("columns names: " + " ".join(dataset.get_column_names()))
    else:
        if args.task == "csv":
            row_count = -1  # the header does not count
            with file(args.input) as lines:
                for line in lines:
                    row_count += 1
                    # print line
            logger.debug("row_count: %d", row_count)
            with file(args.input) as lines:
                line = next(lines).strip()
                # print line
                names = line.strip().split(",")
                line = next(lines).strip()
                values = line.strip().split(",")
                numerics = []
                for value in values:
                    try:
                        float(value)
                        numerics.append(True)
                    except:
                        numerics.append(False)
                names_numeric = [name for name, numeric in zip(names, numerics) if numeric]
                print(names_numeric)
                output = vaex.file.other.Hdf5MemoryMapped.create(args.output, row_count, names_numeric)
                Ncols = len(names)
                cols = [output.columns[name] if numeric else None for name, numeric in zip(names, numerics)]

                def copy(line, row_index):
                    values = line.strip().split(",")
                    for column_index in range(Ncols):
                        if numerics[column_index]:
                            value = float(values[column_index])
                            cols[column_index][row_index] = value
                row = 0
                copy(line, row)
                row += 1
                progressbar = vaex.utils.progressbar(title="exporting") if args.progress else None
                for line in lines:
                    # print line
                    copy(line, row)
                    row += 1
                    if row % 1000:
                        progressbar.update(row / float(row_count))
                progressbar.finish()
                # print names
        else:
            if args.columns:
                columns = args.columns
            else:
                columns = None
            if columns is None:
                columns = dataset.get_column_names(strings=True, virtual=args.virtual)
            for column in columns:
                if column not in dataset.get_column_names(strings=True, virtual=True):
                    if not args.quiet:
                        print("column %r does not exist, run with --list or -l to list all columns" % column)
                    return 1

            base, output_ext = os.path.splitext(args.output)
            if output_ext not in [".hdf5", ".fits", ".arrow"]:
                if not args.quiet:
                    print("extension %s not supported, only .hdf5, .arrow and .fits are" % output_ext)
                return 1

            if not args.quiet:
                print("exporting %d rows and %d columns" % (len(dataset), len(columns)))
                print("columns: " + " ".join(columns))
            progressbar = vaex.utils.progressbar(title="exporting") if args.progress else None

            def update(p):
                if progressbar:
                    progressbar.update(p)
                return True
            if args.filter:
                dataset.select(args.filter, name='export')
                selection = 'export'
            else:
                selection = None
            if output_ext == ".hdf5":
                export_hdf5(dataset, args.output, column_names=columns, progress=update, shuffle=args.shuffle, sort=args.sort, selection=selection)
            elif output_ext == ".arrow":
                from vaex_arrow.export import export as export_arrow
                export_arrow(dataset, args.output, column_names=columns, progress=update, shuffle=args.shuffle, sort=args.sort, selection=selection)
            elif output_ext == ".fits":
                export_fits(dataset, args.output, column_names=columns, progress=update, shuffle=args.shuffle, sort=args.sort, selection=selection)
            if progressbar:
                progressbar.finish()
            if not args.quiet:
                print("\noutput to %s" % os.path.abspath(args.output))
            dataset.close_files()
    return 0