# -*- coding: utf-8 -*-
import collections
from operator import itemgetter
import functools
import time
import copy
import json
import traceback
import numpy as np
from functools import reduce
import threading

from matplotlib.figure import Figure
import matplotlib
import matplotlib.colors
import matplotlib.widgets
import matplotlib.cm
import matplotlib.ticker

import astropy.units

import vaex
import logging
import vaex.events
import vaex.kld
import vaex.ui.plugin.zoom
import vaex.ui.plugin.vector3d
import vaex.ui.plugin.animation
import vaex.ui.plugin.tasks
import vaex.ui.plugin.transferfunction
import vaex.ui.imageblending
import vaex.ui.colormaps
import vaex.ui.templates
import vaex.promise
import vaex.ui.volumerendering

import vaex.ui.qt as dialogs
from vaex.ui.icons import iconfile
import vaex.vaexfast
from vaex.ui import qt, undo

from vaex.utils import AttrDict
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

from vaex.ui.qt import *
if qt_mayor == 5:
    from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
else:
    from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas

grid_resolutions = [32, 64, 128, 256, 512, 1024]
vector_grid_resolutions = [8, 16, 32, 64, 128, 256]

logger = logging.getLogger("vaex.ui.window")

class Slicer(matplotlib.widgets.Widget):
    """
    """

    def __init__(self, plot_window, axes, canvas, useblit=True, horizOn=False, vertOn=True,
                 radius=0.05, **lineprops):

        self.plot_window = plot_window
        self.canvas = canvas
        self.axes = axes
        self.horizOn = horizOn
        self.vertOn = vertOn

        xmin, xmax = axes[-1].get_xlim()
        ymin, ymax = axes[-1].get_ylim()
        xmid = 0.5 * (xmin + xmax)
        ymid = 0.5 * (ymin + ymax)

        self.visible = True
        self.useblit = useblit and self.canvas.supports_blit
        self.background = None
        self.needclear = False
        self.radius = radius

        if self.useblit:
            lineprops['animated'] = True

        import matplotlib.patches

        self.ellipse = matplotlib.patches.Ellipse([xmid, ymid], 1, 1, alpha=0.2, visible=False)
        self.update_ellipse(xmid, ymid)
        axes[0].add_patch(self.ellipse)

        if vertOn:
            self.vlines = [ax.axvline(xmid, visible=False, **lineprops)
                           for ax in axes]
        else:
            self.vlines = []

        if horizOn:
            self.hlines = [ax.axhline(ymid, visible=False, **lineprops)
                           for ax in axes]
        else:
            self.hlines = []

        self.connect()

    def connect(self):
        """connect events"""
        self._cidmotion = self.canvas.mpl_connect('motion_notify_event',
                                                  self.onmove)
        self._ciddraw = self.canvas.mpl_connect('draw_event', self.clear)

    def disconnect(self):
        """disconnect events"""
        self.canvas.mpl_disconnect(self._cidmotion)
        self.canvas.mpl_disconnect(self._ciddraw)

    def clear(self, event):
        """clear the cursor"""
        if self.useblit:
            self.background = (
                self.canvas.copy_from_bbox(self.canvas.figure.bbox))
        for line in self.vlines + self.hlines:
            line.set_visible(False)
        self.ellipse.set_visible(False)

    def onmove(self, event):
        if event.inaxes is None:
            self.plot_window.slice_none()
            return
        if not self.canvas.widgetlock.available(self):
            return
        self.needclear = True
        if not self.visible:
            return
        if self.vertOn:
            for line in self.vlines:
                line.set_xdata((event.xdata, event.xdata))
                line.set_visible(self.visible)
        if self.horizOn:
            for line in self.hlines:
                line.set_ydata((event.ydata, event.ydata))
                line.set_visible(self.visible)
        self.plot_window.slice_circle(event.xdata, event.ydata, self.radius)

        self.update_ellipse(event.xdata, event.ydata)
        self.ellipse.set_visible(True)
        self._update()

    def update_ellipse(self, x, y):
        xlim, ylim = self.plot_window.self.state.ranges_viewport
        width = abs(xlim[1] - xlim[0])
        height = abs(ylim[1] - ylim[0])
        scale = self.radius * 2

        self.ellipse.center = [x, y]
        self.ellipse.width = width * scale
        self.ellipse.height = height * scale

    def _update(self):
        if self.useblit:
            if self.background is not None:
                self.canvas.restore_region(self.background)
            if self.vertOn:
                for ax, line in zip(self.axes, self.vlines):
                    ax.draw_artist(line)
            if self.horizOn:
                for ax, line in zip(self.axes, self.hlines):
                    ax.draw_artist(line)
            for ax in self.axes:
                ax.draw_artist(self.ellipse)
            self.canvas.blit(self.canvas.figure.bbox)
        else:

            self.canvas.draw_idle()

    def disconnect_events(self):
        self.ellipse.set_visible(False)
        self.canvas.restore_region(self.background)
        self.canvas.blit(self.canvas.figure.bbox)
        self.disconnect()

class PlotDialog(QtGui.QWidget):
    """
    :type app: VaexApp
    :type current_layer: LayerTable
    :type layers: list[LayerTable]
    """

    def __init__(self, parent, dataset, dimensions, axisnames, app, width=5, height=4, dpi=100, **options):
        super(PlotDialog, self).__init__()
        self.parent_widget = parent
        self.data_panel = parent
        self.options = options
        self.dataset = dataset
        self.datasets = {}  # keep a record of dataset.path to path for history/state management
        self.app = app
        self.layers = []
        index = len(self.app.windows)
        self.name = options.get("window_name", "%s-%d" % (self.dataset.name, index))
        self.dimensions = dimensions
        if "fraction" in self.options:
            dataset.set_active_fraction(float(self.options["fraction"]))

        self.state = AttrDict()
        self.state.xlabel = options.get("xlabel")
        self.state.ylabel = options.get("ylabel")

        self.enable_slicing = options.get("enable_slicing", False)

        self.menu_bar = QtGui.QMenuBar(self)
        self.menu_file = QtGui.QMenu("&File", self.menu_bar)
        self.menu_bar.addMenu(self.menu_file)
        self.menu_view = QtGui.QMenu("&View", self.menu_bar)
        self.menu_bar.addMenu(self.menu_view)
        self.menu_mode = QtGui.QMenu("&Mode", self.menu_bar)
        self.menu_bar.addMenu(self.menu_mode)
        self.menu_selection = QtGui.QMenu("&Selection", self.menu_bar)
        self.menu_bar.addMenu(self.menu_selection)

        self.menu_samp = QtGui.QMenu("SAM&P", self.menu_bar)
        self.menu_bar.addMenu(self.menu_samp)

        self.undoManager = parent.undoManager
        self.full_state_history = []
        self.full_state_history_index = -1
        self.full_state_history_change_reason = None
        self.setWindowTitle(self.name)
        self.dataset = dataset
        self.axisnames = axisnames
        # self.expressions = expressions
        # self.dimensions = len(self.expressions)
        # self.grids = Grids(self.dataset, self.pool, *self.expressions)

        if self.dimensions == 3:
            self.resize(800 + 400, 700)
        else:
            self.resize(800 + 150, 700)

        self.plugin_queue_toolbar = []  # list of tuples (callback, order)
        self.plugins = [cls(self) for cls in vaex.ui.plugin.PluginPlot.registry if cls.useon(self.__class__)]
        self.plugins_map = {plugin.name: plugin for plugin in self.plugins}
        # self.plugin_zoom = plugin.zoom.ZoomPlugin(self)

        self.state.aspect = None
        self.state.axis_lock = False

        self.update_counter = 0
        self.t_0 = 0
        self.t_last = 0
        self.slice_radius = 0.05

        self.shortcuts = []
        self.messages = {}

        default_grid_size = 256 if self.dimensions == 2 else 128
        self.state.grid_size = eval(self.options.get("grid_size", str(default_grid_size)))
        self.state.vector_grid_size = eval(self.options.get("vector_grid_size", "16"))

        self.fig = Figure(figsize=(width, height), dpi=dpi)
        self.canvas = FigureCanvas(self.fig)
        self.canvas.setParent(self)
        self.add_axes()

        # if an update is scheduled, cancel any replot, since the preperations for the update may leave the grids
        # in the layers in a undefined state
        self.queue_redraw = Queue("redraw", 5, self.canvas.draw)  # only draw the canvas again
        self.queue_replot = Queue("replot", 10, self.plot)  # redo the whole plot, but no computation

        def pre():
            self.queue_replot.cancel()
            for layer in self.layers:
                layer.cancel_tasks()
        self.queue_update = Queue("update", 100, self.update_direct, pre=pre)  # a complete recalculation and refresh of the plot

        self.layout_main = QtGui.QVBoxLayout()
        self.layout_content = QtGui.QHBoxLayout()
        self.layout_main.setContentsMargins(0, 0, 0, 0)
        self.layout_content.setContentsMargins(0, 0, 0, 0)
        self.layout_main.setSpacing(0)
        self.layout_content.setSpacing(0)
        self.layout_main.addWidget(self.menu_bar)

        # self.button_layout.setSpacing(0)

        self.boxlayout = QtGui.QVBoxLayout()
        self.boxlayout_right = QtGui.QVBoxLayout()
        self.boxlayout.setContentsMargins(0, 0, 0, 0)
        self.boxlayout_right.setContentsMargins(0, 0, 0, 0)

        # self.ranges = [None for _ in range(self.dimensions)] # min/max for the data
        self.state.ranges_viewport = [None for _ in range(self.dimensions)]  # min/max for the plots
        self.state.range_level_show = None
        # if zooming in with for instance pinching, we like to know what the ranges were
        # before zooming for the undo, and not all ranges in between
        self.last_ranges_viewport = None
        self.last_range_level_show = None

        self._plot_event = None  # used for unittesting only

        # self.ranges_previous = None
        # self.state.ranges_viewport_previous = None
        # self.ranges_level_previous = None

        self.currentModes = None
        self.lastAction = None

        self.beforeCanvas(self.layout_main)
        self.layout_main.addLayout(self.layout_content, 1.)
        self.layout_plot_region = QtGui.QHBoxLayout()
        self.layout_plot_region.addWidget(self.canvas, 1)

        self.boxlayout.addLayout(self.layout_plot_region, 1)
        self.addToolbar2(self.layout_main)
        self.afterCanvas(self.boxlayout_right)
        self.layout_content.addLayout(self.boxlayout, 1.)

        self.status_bar = QtGui.QStatusBar(self)
        self.button_cancel = QtGui.QToolButton(self.status_bar)
        self.button_cancel.setText("cancel")
        self.button_cancel.setContentsMargins(0, 0, 0, 0)
        self.status_bar.layout().setContentsMargins(0, 0, 0, 0)
        self.status_bar.layout().setSpacing(0)

        self.progress_bar = QtGui.QProgressBar(self.status_bar)
        self.progress_bar.setMaximum(1000)
        self.progress_bar.setMinimumWidth(100)
        self.progress_bar.setFixedWidth(100)
        self.button_cancel.setEnabled(False)
        self.progress_start_time = time.time()

        self.label_time = QtGui.QLabel("", self.toolbar)

        # index = self.status_bar.
        self.status_bar.addPermanentWidget(self.progress_bar)
        self.status_bar.addPermanentWidget(self.button_cancel)
        self.status_bar.addPermanentWidget(self.label_time)

        def on_click_cancel():
            self.button_cancel.setEnabled(False)
            for layer in self.layers:
                layer.cancel_tasks()
        self.button_cancel.clicked.connect(on_click_cancel)

        self.layout_main.addWidget(self.status_bar)

        self.layout_content.addLayout(self.boxlayout_right, 0)
        self.setLayout(self.layout_main)

        # self.plot()
        FigureCanvas.setSizePolicy(self,
                                   QtGui.QSizePolicy.Expanding,
                                   QtGui.QSizePolicy.Expanding)
        FigureCanvas.updateGeometry(self)
        self.currentMode = None
        self.shortcuts = []

        self.grabGesture(QtCore.Qt.PinchGesture)
        self.grabGesture(QtCore.Qt.PanGesture)
        self.grabGesture(QtCore.Qt.SwipeGesture)

        self.signal_samp_send_selection = vaex.events.Signal("samp send selection")
        self.signal_closed = vaex.events.Signal("close plot window")

        self.signal_plot_finished = vaex.events.Signal("plot finished")

        self.canvas.mpl_connect('resize_event', self.on_resize_event)
        self.canvas.mpl_connect('motion_notify_event', self.onMouseMove)
        # self.pinch_self.state.ranges_viewport = [None for i in range(self.dimension)]

    def get_label(self, axis=0):
        if self.layers:
            layer_labels = [layer.state.labels[axis] for layer in self.layers]
            if any(layer_labels):
                return ",".join([label for label in layer_labels if label])
            else:
                return self.get_default_label(axis=axis)
        else:
            return ""

    def get_default_label(self, axis=0):
        if len(self.layers) == 0:
            return ""
        else:
            first_layer = self.layers[0]
            if axis == 0:
                expr = first_layer.x
            if axis == 1:
                if self.dimensions > 1:
                    expr = first_layer.y
                else:
                    return first_layer.amplitude_label()
            if axis == 2:
                expr = first_layer.z
            label = expr
            unit = first_layer.dataset.unit(expr)
            try:  # if we can convert the unit, use that for the labeling
                output_unit = astropy.units.Unit(first_layer.state.output_units[axis])
                if first_layer.state.output_units[axis] and unit:  # avoid unnecessary error msg'es
                    output_unit.to(unit)
                    unit = output_unit
            except:
                logger.exception("unit error")
            logger.debug("unit: %r", unit)
            if unit is not None:
                label = "%s (%s)" % (label, unit.to_string('latex_inline'))
            return label

    def set_plot_size(self, width, height):
        # use for testing only, to get a fixed size per platform
        self.canvas.setFixedSize(width, height)

    def set_layer_progress(self, layer, fraction):
        if fraction == 0:
            self.progress_start_time = time.time()
        if fraction == 1:
            delta_time = time.time() - self.progress_start_time
            info = "%.2f sec" % delta_time
            self.label_time.setText(info)
        elif fraction > 0:
            delta_time = time.time() - self.progress_start_time
            estimated = delta_time / fraction - delta_time
            info = "ETA: %.2f sec" % estimated
            self.label_time.setText(info)

        self.button_cancel.setEnabled(True)
        fraction = self.get_progress_fraction()
        self.progress_bar.setValue(fraction * 1000)
        QtCore.QCoreApplication.instance().processEvents()

    def get_progress_fraction(self):
        total_fraction = 0
        updating_layers = [layer for layer in self.layers]  # if layer.tasks]
        total_fraction = sum([layer.get_progress_fraction() for layer in updating_layers])
        return total_fraction / len(updating_layers)

    def slice_none(self):
        mask = np.ones((self.state.grid_size,) * self.dimensions, dtype=np.bool)
        layer = self.current_layer
        if layer:
            layer.signal_slice_change.emit(mask, False)

    def slice_circle(self, x, y, radius):
        logger.debug("slice circle: %r %r", x, y)
        xlim, ylim = self.state.ranges_viewport
        width = abs(xlim[1] - xlim[0])
        height = abs(ylim[1] - ylim[0])
        xrel = (x - xlim[0]) / width
        yrel = (y - ylim[0]) / height

        import vaex.utils
        x = vaex.utils.linspace_centers(0, 1, self.state.grid_size)
        y = vaex.utils.linspace_centers(0, 1, self.state.grid_size)
        x, y = np.meshgrid(x, y)
        distance = np.sqrt((x - xrel)**2 + (y - yrel)**2)
        mask = distance < radius
        layer = self.current_layer
        if layer:
            layer.signal_slice_change.emit(mask, False)

    def update_all_layers(self):
        for layer in self.layers:
            layer.flag_needs_update()
        self.queue_update()

    def add_axes(self):
        self.axes = self.fig.add_subplot(111)
        self.axes.xaxis_index = 0
        if self.dimensions > 1:
            self.axes.yaxis_index = 1
        if int(matplotlib.__version__.split('.')[0]) < 3:
            self.axes.hold(True)

    def plug_toolbar(self, callback, order):
        self.plugin_queue_toolbar.append((callback, order))

    def plug_page(self, callback, pagename, pageorder, order):
        self.plugin_queue_page.append((callback, pagename, pageorder, order))

    def plug_grids(self, callback_define, callback_draw):
        self.plugin_grids_defines.append(callback_define)
        self.plugin_grids_draw.append(callback_draw)

    def getAxesList(self):
        return [self.axes]

    def __repr__(self):
        return "<%s at 0x%x layers=%r>" % (self.__class__.__name__, id(self), self.layers)

    def get_options(self):
        options = collections.OrderedDict()
        options["grid_size"] = self.state.grid_size
        options["vector_grid_size"] = self.state.vector_grid_size
        options["ranges_viewport"] = self.state.ranges_viewport
        options["aspect"] = self.state.aspect
        layer = self.current_layer
        if layer is not None:
            options["layer"] = layer.get_options()
        options = copy.deepcopy(options)
        return dict(options)

    def apply_options(self, options, update=True):
        # map = {"expressions",}
        # recognize = "self.state.ranges_viewport grid_size vector_grid_size aspect".split()
        recognize = "ranges_viewport  aspect".split()
        for key in recognize:
            if key in list(options.keys()):
                value = options[key]
                setattr(self, key, copy.copy(value))
                if key == "aspect":
                    self.action_aspect_lock_one.setChecked(bool(value))
        for plugin in self.plugins:
            plugin.apply_options(options)
        for key in list(options.keys()):
            if key not in recognize:
                logger.error("option %s not recognized, ignored" % key)
        layer = self.current_layer
        if layer is not None:
            layer.apply_options(options["layer"], update=False)
        if update:
            self.queue_update()

    def load_options(self, name):
        self.plugins_map["favorites"].load_options(name, update=False)

    def select_layer(self, index):
        self.layer_selection.setCurrentIndex(index + 1)

    def add_layer(self, expressions, dataset=None, name=None, **options):
        if dataset is None:
            dataset = self.dataset
        if name is None:
            name = options.get("layer_name", "Layer: " + str(len(self.layers) + 1))
        self.datasets[dataset.path] = dataset
        ranges = copy.deepcopy(self.state.ranges_viewport)
        logger.debug("adding layer {name} with expressions {expressions} for dataset {dataset} and options {options}".format(**locals()))

        if len(self.layers) > 0:
            first_layer = self.layers[0]
            assert len(expressions) == first_layer.dimensions
            for i in range(self.dimensions):
                if ranges[i] is None and first_layer.state.ranges_grid[i] is not None:
                    ranges[i] = copy.copy(first_layer.state.ranges_grid[i])
        layer = vaex.ui.layers.LayerTable(self, name, dataset, expressions, self.axisnames, options, self.fig, self.canvas, ranges)
        self.layers.append(layer)
        layer.build_widget_qt(self.widget_layer_stack)  # layer.widget is the widget build
        self.widget_layer_stack.addWidget(layer.widget)
        self.queue_history_change("added layer: " + name)

        # layer.build_widget_qt_layer_control(self.frame_layer_controls)
        # self.layout_frame_layer_controls.addWidget(layer.widget_layer_control)

        layer.widget.setVisible(False)
        self.layer_selection.addItem(name + "(%s)" % dataset.name)
        # self.layer_selection.setCurrentIndex(len(self.layers))
        self.select_layer(len(self.layers) - 1)

        def on_expression_change(layer, axis_index, expression):
            if not self.state.axis_lock:  # and len(self.layers) == 1:
                self.state.ranges_viewport[axis_index] = None
            self.compute()
            error_text = self.dataset.executor.execute()
            if error_text:
                dialog_error(self, "Error in expression", "Error: " + error_text)

        def on_plot_dirty(layer=None):
            logger.debug("received signal plot dirty, layer=%r" % layer)
            self.queue_replot()

        layer.signal_expression_change.connect(on_expression_change)
        layer.signal_plot_dirty.connect(on_plot_dirty)
        layer.signal_plot_update.connect(self.queue_update)

        if "options" in options:
            assert self.current_layer == layer
            self.load_options(options["options"])
        # $layer.add_jobs()
        self.queue_update(layer=layer)
        # self.dataset.executor.execute()
        # self.queue_update()
        logger.debug("added layer")
        return layer

    def _wait(self):
        """Used for unittesting to make sure the plots are all done"""
        logger.debug("will wait for last plot to finish")
        self._plot_event = threading.Event()
        self.queue_update._wait()
        self.queue_replot._wait()
        self.queue_redraw._wait()
        qt_app = QtCore.QCoreApplication.instance()
        sleep = 10
        while not self._plot_event.is_set():
            logger.debug("waiting for last plot to finish")
            qt_app.processEvents()
            QtTest.QTest.qSleep(sleep)
        logger.debug("waiting for plot finished")
        # traceback.print_stack()
        # self._plot_event.wait()
        # self._plot_event.wait()

    def plot_to_png(self, filename=None):
        if filename is None:
            import tempfile
            handle, filename = tempfile.mkstemp(".png")
            logger.debug("write to %s" % filename)
        self.fig.savefig(filename)
        return filename

    def on_resize_event(self, event):
        if not self.action_mini_mode_ultra.isChecked():
            logger.debug("resize event")
            self.fig.tight_layout()
            self.queue_redraw()

    def event(self, event):
        if isinstance(event, QtGui.QGestureEvent):
            for gesture in event.activeGestures():
                if isinstance(gesture, QtGui.QPinchGesture):
                    center = gesture.centerPoint()
                    x, y = center.x(), center.y()
                    geometry = self.canvas.geometry()
                    if geometry.contains(x, y):
                        rx = x - geometry.x()
                        ry = y - geometry.y()
                        # print self.canvas.geometry, self.canvas.mouse_grabber
                        axes_list = [ax for ax in self.getAxesList() if ax.contains_point((rx, geometry.height() - 1 - ry))]
                        # nx, ny = rx/geometry.width(), y/geometry.height()
                        if len(axes_list) > 0:
                            axes = axes_list[0]
                            transform = axes.transData.inverted().transform
                            x_data, y_data = transform([rx, geometry.height() - 1 - ry])
                            if gesture.lastScaleFactor() != 0:
                                scale = (gesture.scaleFactor() / gesture.lastScaleFactor())
                            else:
                                scale = (gesture.scaleFactor())
                            # @scale = gesture.totalScaleFactor()
                            scale = 1. / (scale)
                            self.zoom(scale, axes, x_data, y_data)
            return True
        else:
            return super(PlotDialog, self).event(event)
            # return True

    def closeEvent(self, event):
        # disconnect this event, otherwise we get an update/redraw for nothing
        # since closing a dialog causes this event to fire otherwise
        # self.parent_widget.plot_dialogs.remove(self)

        # self.dataset.executor.signal_begin.disconnect(self._begin_signal)
        # self.dataset.executor.signal_progress.disconnect(self._progress_signal)
        # self.dataset.executor.signal_end.disconnect(self._end_signal)
        # self.dataset.executor.signal_cancel.disconnect(self._cancel_signal)

        for layer in self.layers:
            layer.removed()
        for plugin in self.plugins:
            plugin.clean_up()
        super(PlotDialog, self).closeEvent(event)
        self.signal_closed.emit(self)

    def getExpressionList(self):
        return self.dataset.column_names

    def add_pages(self, toolbox):
        pass

    def fill_menu_layer_new(self):
        self.menu_layer_new.clear()
        for dataset in self.data_panel.dataset_list:
            menu_dataset = QtGui.QMenu(dataset.name, self.menu_layer_new)
            self.menu_layer_new.addMenu(menu_dataset)
            for column1 in dataset.get_column_names(virtual=True):
                if self.dimensions == 1:
                    action_col1 = QtGui.QAction(column1, menu_dataset)
                    menu_dataset.addAction(action_col1)

                    def add_layer_1(ignore=None, column1=column1, dataset=dataset):
                        self.add_layer([column1], dataset=dataset)
                        dataset.executor.execute()
                    action_col1.triggered.connect(add_layer_1)
                else:
                    menu_col1 = QtGui.QMenu(column1, menu_dataset)
                    menu_dataset.addMenu(menu_col1)
                    for column2 in dataset.get_column_names(virtual=True):
                        if self.dimensions == 2:
                            action_col2 = QtGui.QAction(column2, menu_dataset)
                            menu_col1.addAction(action_col2)

                            def add_layer_2(_ignore=None, column1=column1, column2=column2, dataset=dataset):
                                self.add_layer([column1, column2], dataset=dataset)
                                dataset.executor.execute()
                            action_col2.triggered.connect(add_layer_2)
                        else:
                            pass

                    # action = QtGui.QAction()
    def remove_layer(self, layer=None, layer_index=None):
        if layer is not None:
            layer = layer
        elif layer_index is not None:
            layer = self.layers[layer_index]
        else:
            layer = self.current_layer
        logger.debug("remove layer: %r" % layer)
        if layer is not None:
            index = self.layers.index(layer)
            layer.removed()
            self.layers.remove(layer)
            self.layer_selection.removeItem(index + 1)  # index 0 is the layer control
            self.widget_layer_stack.removeWidget(layer.widget)
            self.queue_history_change("Remove layer: %s" % layer.name)
            self.queue_push_full_state()
            self.plot()

    def afterCanvas(self, layout):

        layout.setContentsMargins(0, 0, 0, 0)
        layout.setSpacing(0)

        self.layer_box = QtGui.QGroupBox("Layers", self)
        self.layout_layer_box = QtGui.QVBoxLayout()
        self.layout_layer_box.setSpacing(0)
        self.layout_layer_box.setContentsMargins(0, 0, 0, 0)
        self.layer_box.setLayout(self.layout_layer_box)

        self.layout_layer_buttons = QtGui.QHBoxLayout()

        self.button_layer_new = QtGui.QToolButton(self)
        self.button_layer_new.setIcon(QtGui.QIcon(iconfile("layer--plus")))
        self.button_layer_new.setPopupMode(QtGui.QToolButton.InstantPopup)
        self.button_layer_new.setToolButtonStyle(QtCore.Qt.ToolButtonTextBesideIcon)

        toolbuttonSizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Preferred, QtGui.QSizePolicy.Fixed)
        self.button_layer_new.setSizePolicy(toolbuttonSizePolicy)

        self.button_layer_new.setText("add")
        self.button_layer_new.setEnabled(self.dimensions < 3)
        self.menu_layer_new = QtGui.QMenu()
        self.fill_menu_layer_new()
        self.button_layer_new.setMenu(self.menu_layer_new)
        self.button_layer_delete = QtGui.QToolButton(self)
        self.button_layer_delete.setIcon(QtGui.QIcon(iconfile("layer--minus")))
        self.button_layer_delete.setText("remove")
        self.button_layer_delete.setToolButtonStyle(QtCore.Qt.ToolButtonTextBesideIcon)
        self.button_layer_delete.setSizePolicy(toolbuttonSizePolicy)
        self.button_layer_delete.setEnabled(self.dimensions < 3)

        def on_layer_remove(_ignore=None):
            logger.debug("remove layer")
            self.remove_layer()
        self.button_layer_delete.clicked.connect(on_layer_remove)
        self.layout_layer_buttons.addWidget(self.button_layer_new, 0)
        self.layout_layer_buttons.addWidget(self.button_layer_delete, 0)

        self.layer_selection = QtGui.QComboBox(self)
        self.layer_selection.addItems(["Layer controls"])
        self.layout_layer_box.addLayout(self.layout_layer_buttons)
        self.layout_layer_box.addWidget(self.layer_selection)
        self.current_layer = None

        def onSwitchLayer(index):
            logger.debug("switch to layer: %r %r" % (index, self.layers))
            # if self.current_layer is not None:
            #   self.bottom_layout.removeWidget(self.current_layer.widget)
            #   self.current_layer.widget.setVisible(False)
            # self.current_layer = self.layers[index]
            # self.current_layer.widget.setVisible(True)
            # self.bottom_layout.addWidget(self.current_layer.widget)
            # self.bottom_layout.update()
            # self.bottomFrame.updateGeometry()
            layer_index = index - 1
            if index == 0:
                self.current_layer = None
                for layer in self.layers:
                    layer_control_widget = layer.grab_layer_control(self.frame_layer_controls)
                    self.layout_frame_layer_controls.addWidget(layer_control_widget)
                    # layer.build_widget_qt_layer_control(self.frame_layer_controls)
                    #
            else:
                self.layers[layer_index].release_layer_control(self.frame_layer_controls)
                self.current_layer = self.layers[index - 1]
            self.update_favorite_selections()

            self.widget_layer_stack.setCurrentIndex(index)
            # self.widget_layer_stack.setCurrentIndex(0)

        self.layer_selection.currentIndexChanged.connect(onSwitchLayer)

        self.bottomFrame = QtGui.QFrame(self)
        layout.addWidget(self.bottomFrame, 0)

        self.bottom_layout = QtGui.QVBoxLayout()
        self.bottom_layout.setContentsMargins(0, 0, 0, 0)
        self.bottom_layout.setSpacing(0)

        self.bottomFrame.setLayout(self.bottom_layout)
        self.bottom_layout.addWidget(self.layer_box)

        self.widget_layer_stack = QtGui.QStackedWidget(self)
        self.bottom_layout.addWidget(self.widget_layer_stack)

        self.frame_layer_controls = QtGui.QGroupBox("Layer controls", self.widget_layer_stack)
        self.layout_frame_layer_controls = QtGui.QVBoxLayout(self.frame_layer_controls)
        self.layout_frame_layer_controls.setAlignment(QtCore.Qt.AlignTop)
        self.frame_layer_controls.setLayout(self.layout_frame_layer_controls)
        self.widget_layer_stack.addWidget(self.frame_layer_controls)

        self.frame_layer_controls_result = QtGui.QGroupBox("Layer result", self.frame_layer_controls)
        self.layout_frame_layer_controls_result = QtGui.QGridLayout()
        self.frame_layer_controls_result.setLayout(self.layout_frame_layer_controls_result)
        self.layout_frame_layer_controls_result.setSpacing(0)
        self.layout_frame_layer_controls_result.setContentsMargins(0, 0, 0, 0)
        self.layout_frame_layer_controls.addWidget(self.frame_layer_controls_result)

        row = 0
        attr_name = "layer_brightness"
        self.layer_brightness = 1.
        self.slider_layer_brightness = Slider(self.frame_layer_controls_result, "brightness", 10**-1, 10**1, 1000, attrgetter(self, attr_name), attrsetter(self, attr_name), uselog=True, update=self.plot)
        row = self.slider_layer_brightness.add_to_grid_layout(row, self.layout_frame_layer_controls_result)
        attr_name = "layer_gamma"
        self.layer_gamma = 1.
        self.slider_layer_gamma = Slider(self.frame_layer_controls_result, "gamma", 10**-1, 10**1, 1000, attrgetter(self, attr_name), attrsetter(self, attr_name), uselog=True, update=self.plot)
        row = self.slider_layer_gamma.add_to_grid_layout(row, self.layout_frame_layer_controls_result)
        # self.frame_layer_controls_result

        self.blend_modes = list(vaex.ui.imageblending.modes)
        self.blend_mode = self.blend_modes[0]
        self.option_layer_blend_mode = Option(self.frame_layer_controls_result, "blend", self.blend_modes, getter=attrgetter(self, "blend_mode"), setter=attrsetter(self, "blend_mode"), update=self.plot)
        row = self.option_layer_blend_mode.add_to_grid_layout(row, self.layout_frame_layer_controls_result)

        self.background_colors = ["auto", "white", "black"]
        self.background_color = self.background_colors[0]
        self.option_layer_background_color = Option(self.frame_layer_controls_result, "background", self.background_colors, getter=attrgetter(self, "background_color"), setter=attrsetter(self, "background_color"), update=self.plot)
        row = self.option_layer_background_color.add_to_grid_layout(row, self.layout_frame_layer_controls_result)

        # row = self.checkbox_intensity_as_opacity.add_to_grid_layout(row, self.layout_layer_control)

        # self.checkbox_intensity_as_opacity = Checkbox(self.group_box_layer_control, "use_intensity", getter=attrgetter(self, "use_intensity"), setter=attrsetter(self, "use_intensity"), update=self.plot)
        # row = self.checkbox_intensity_as_opacity.add_to_grid_layout(row, self.layout_layer_control)

    def add_shortcut(self, action, key):
        def trigger(action):
            def call(action=action):
                action.toggle()
                action.trigger()
            return call
        if action.isEnabled():
            shortcut = QtGui.QShortcut(QtGui.QKeySequence(key), self)
            shortcut.activated.connect(trigger(action))
            self.shortcuts.append(shortcut)

    def checkUndoRedo(self):
        can_undo = self.full_state_history_index >= 1
        can_redo = self.full_state_history_index < (len(self.full_state_history) - 1)
        # if len(full_state_history)
        self.action_undo.setEnabled(can_undo)
        if can_undo:
            reason, state = self.full_state_history[self.full_state_history_index]
            self.action_undo.setToolTip("Undo: " + reason)

        self.action_redo.setEnabled(can_redo)
        if can_redo:
            reason, state = self.full_state_history[self.full_state_history_index + 1]
            self.action_redo.setToolTip("Redo: " + reason)

    def onActionUndo(self):
        logger.debug("undo")
        # self.undoManager.undo()
        self.full_state_history_index -= 1
        reason, state = self.full_state_history[self.full_state_history_index]
        logger.debug("restore state")
        self.restore_full_state(state)
        logger.debug("state restored")
        self.checkUndoRedo()
        logger.debug("undo/redo buttons checked")

    def onActionRedo(self):
        logger.debug("redo")
        self.full_state_history_index += 1
        reason, state = self.full_state_history[self.full_state_history_index]
        logger.debug("restore state")
        self.restore_full_state(state)
        logger.debug("state restored")
        # self.undoManager.redo()
        self.checkUndoRedo()
        logger.debug("undo/redo buttons checked")

    def onMouseMove(self, event):
        x, y = event.xdata, event.ydata
        if x is not None:
            extra_text = self.getExtraText(x, y)
            # extra_text = "TODO:"
            if extra_text:
                self.message("x,y=%5.4e,%5.4e %s" % (x, y, extra_text), index=0)
            else:
                self.message("x,y=%5.4e,%5.4e" % (x, y), index=0)
        else:
            self.message(None)

    def getExtraText(self, x, y):
        layer = self.current_layer
        if hasattr(layer, "amplitude_grid"):
            amplitude = layer.amplitude_grid
            if len(amplitude.shape) == 1:
                    # if self.ranges[0]:
                N = amplitude.shape[0]
                if layer.state.ranges_grid[0] is not None:
                    xmin, xmax = layer.state.ranges_grid[0]
                    index = (x - xmin) / (xmax - xmin) * N
                    if index >= 0 and index < N:
                        index = int(index)
                        return "value = %f" % (amplitude[index])
            if len(amplitude.shape) == 2:
                # if self.ranges[0] and self.ranges[1]:
                Nx, Ny = amplitude.shape
                if layer.state.ranges_grid[0] is not None and layer.state.ranges_grid[1] is not None:
                    xmin, xmax = layer.state.ranges_grid[0]
                    ymin, ymax = layer.state.ranges_grid[1]
                    xindex = (x - xmin) / (xmax - xmin) * Nx
                    yindex = (y - ymin) / (ymax - ymin) * Ny
                    if xindex >= 0 and xindex < Nx and yindex >= 0 and yindex < Nx:
                        return "value = %f" % (amplitude[int(yindex), int(xindex)])

    def message(self, text, index=0):
        if text is None:
            if index in self.messages:
                del self.messages[index]
        else:
            self.messages[index] = text
        text = ""
        keys = list(self.messages.keys())
        keys.sort()
        text_parts = [self.messages[key] for key in keys]
        self.status_bar.showMessage(" | ".join(text_parts))

    def beforeCanvas(self, layout):
        self.addToolbar(layout)  # , yselect=True, lasso=False)

    def setMode(self, action, force=False):
        logger.debug("set mode %r %r %r" % (action, action.text(), action.isChecked()))
        # if not (action.isChecked() or force):
        if not action.isEnabled():
            logger.error("action selected that was disabled: %r" % action)
            self.setMode(self.lastAction)
            return
        if not (action.isChecked()):
            logger.debug("ignore action")
        else:
            self.lastAction = action
            axes_list = self.getAxesList()
            if self.currentModes is not None:
                logger.debug("disconnect %r" % (self.currentModes, ))
                for mode in self.currentModes:
                    mode.disconnect_events()
                    mode.active = False
            useblit = True
            if action == self.action_move:
                self.currentModes = [Mover(self, axes) for axes in axes_list]
            if action == self.action_pick:
                # hasy = hasattr(self, "getdatay")
                # hasx = hasattr(self, "getdatax")
                layer = self.current_layer
                if layer is not None:
                    hasx = True
                    hasy = len(layer.state.expressions) > 1
                    self.currentModes = [matplotlib.widgets.Cursor(axes, hasy, hasx, color="red", linestyle="dashed", useblit=useblit) for axes in axes_list]
                    for cursor in self.currentModes:
                        def onmove(event, current=cursor, cursors=self.currentModes):
                            if event.inaxes:
                                for other_cursor in cursors:
                                    if current != other_cursor:
                                        other_cursor.onmove(event)
                        cursor.connect_event('motion_notify_event', onmove)
                    if hasx and hasy:
                        for mode in self.currentModes:
                            mode.connect_event('button_press_event', self.onPickXY)
                    elif hasx:
                        for mode in self.currentModes:
                            mode.connect_event('button_press_event', self.onPickX)
                    elif hasy:
                        for mode in self.currentModes:
                            mode.connect_event('button_press_event', self.onPickY)
                if useblit:
                    self.canvas.draw()  # buggy otherwise
            if action == self.action_xrange:
                logger.debug("setting last select action to xrange")
                self.lastActionSelect = self.action_xrange
                self.currentModes = [matplotlib.widgets.SpanSelector(axes, functools.partial(self.onSelectX, axes=axes), 'horizontal', useblit=useblit) for axes in axes_list]
                if useblit:
                    self.canvas.draw()  # buggy otherwise
            if action == self.action_yrange:
                logger.debug("setting last select action to yrange")
                self.lastActionSelect = self.action_yrange
                self.currentModes = [matplotlib.widgets.SpanSelector(axes, functools.partial(self.onSelectY, axes=axes), 'vertical', useblit=useblit) for axes in axes_list]
                if useblit:
                    self.canvas.draw()  # buggy otherwise
            if action == self.action_lasso:
                logger.debug("setting last select action to lasso")
                self.lastActionSelect = self.action_lasso
                self.currentModes = [matplotlib.widgets.LassoSelector(axes, functools.partial(self.onSelectLasso, axes=axes)) for axes in axes_list]
                if useblit:
                    self.canvas.draw()  # buggy otherwise
            if action == self.action_select_rectangle:
                logger.debug("setting last select action to rect")
                self.lastActionSelect = self.action_select_rectangle
                self.currentModes = [matplotlib.widgets.RectangleSelector(axes, functools.partial(self.on_select_rectangle, axes=axes), spancoords='data') for axes in axes_list]
                if useblit:
                    self.canvas.draw()  # buggy otherwise
            if action == self.action_slice:
                logger.debug("setting mode to slice")
                # self.lastActionSelect = self.action_slice
                self.currentModes = [Slicer(self, [axes], canvas=self.canvas, horizOn=False, vertOn=False, radius=self.slice_radius) for axes in axes_list]
                if useblit:
                    self.canvas.draw()  # buggy otherwise
            # self.plugin_zoom.setMode(action)
            for plugin in self.plugins:
                logger.debug("plugin %r %r.setMode" % (plugin, plugin.name))
                plugin.setMode(action)
        self.syncToolbar()

        # if self.action_lasso
        # pass
        # self.

    def onPickX(self, event):
        x, y = event.xdata, event.ydata
        self.selected_point = None

        layer = self.current_layer
        axes = event.inaxes
        logger.debug("pickx %r %r" % (layer, axes))
        if layer is not None and axes is not None:
            layer.coordinates_picked_row = None
            promise = layer.subspace.nearest([x])
            layer.dataset.executor.execute()

            def set(value):
                if value is None:
                    logger.error("could not find nearest")
                else:
                    index, distance, point = value
                    layer.dataset.set_current_row(index)
            promise.then(set).end()
            self.setMode(self.lastAction)

    def onPickY(self, event):
        # TODO: not sure when we want this?
        x, y = event.xdata, event.ydata
        self.selected_point = None

        layer = self.current_layer
        axes = event.inaxes
        logger.debug("pickx %r %r" % (layer, axes))
        if layer is not None and axes is not None:
            layer.coordinates_picked_row = None
            promise = layer.subspace.nearest([y])
            layer.dataset.executor.execute()

            def set(value):
                if value is None:
                    logger.error("could not find nearest")
                else:
                    index, distance, point = value
                    layer.dataset.set_current_row(index)
            promise.then(set).end()
            self.setMode(self.lastAction)

    def onPickXY(self, event):
        x, y = event.xdata, event.ydata
        wx = self.state.ranges_viewport[0][1] - self.state.ranges_viewport[0][0]
        wy = self.state.ranges_viewport[1][1] - self.state.ranges_viewport[1][0]

        x, y = event.xdata, event.ydata
        self.selected_point = None

        layer = self.current_layer
        axes = event.inaxes
        logger.debug("pickx %r %r" % (layer, axes))
        if layer is not None and axes is not None:
            layer.coordinates_picked_row = None
            promise = layer.subspace.nearest([x, y], metric=[1. / wx, 1. / wy])
            layer.dataset.executor.execute()

            def set(value):
                if value is None:
                    logger.error("could not find nearest")
                else:
                    index, distance, point = value
                    layer.dataset.set_current_row(index)
            promise.then(set).end()
        return

    def select(self, name, vmin, vmax):
        values = [vmin, vmax]
        vmin, vmax = min(values), max(values)

        for layer in self.active_layers():
            # xaxis is stored in the matplotlib object / axes.xaxis_index
            expr = getattr(layer, name)
            boolean_expression = "((%s) >= %f) & ((%s) < %f)" % (expr, vmin, expr, vmax)
            logger.debug("expression: %s", boolean_expression)
            layer.dataset.select(boolean_expression, self.select_mode)

    def onSelectX(self, xmin, xmax, axes):
        self.select("x", xmin, xmax)

    def onSelectY(self, ymin, ymax, axes):
        self.select("y", ymin, ymax)

    def onSelectLasso(self, vertices, axes):
        x, y = np.array(vertices).T

        x = np.ascontiguousarray(x, dtype=np.float64)
        y = np.ascontiguousarray(y, dtype=np.float64)
        for layer in self.active_layers():
            layer.dataset.select_lasso(layer.state.expressions[axes.xaxis_index], layer.state.expressions[axes.yaxis_index], x, y, mode=self.select_mode)
            meanx = x.mean()
            meany = y.mean()
            mask = layer.dataset.mask
            self.checkUndoRedo()
            self.queue_update()
        return

    def on_select_rectangle(self, pos1, pos2, axes):
        layer = self.current_layer
        if layer is not None:
            xmin = min(pos1.xdata, pos2.xdata)
            xmax = max(pos1.xdata, pos2.xdata)
            ymin = min(pos1.ydata, pos2.ydata)
            ymax = max(pos1.ydata, pos2.ydata)
            args = (layer.x, xmin, layer.x, xmax, layer.y, ymin, layer.y, ymax)
            expression = "((%s) >= %f) & ((%s) <= %f) & ((%s) >= %f) & ((%s) <= %f)" % args
            logger.debug("rectangle selection using expression: %r" % expression)
            layer.dataset.select(expression, mode=self.select_mode)
            self.checkUndoRedo()
            self.queue_update()
        return

    def set_ranges(self, axis_indices, ranges_viewport=None, range_level=None, add_to_history=False, reason=None):
        if add_to_history:
            action = undo.ActionZoom(self.undoManager, reason or "Change in ranges",
                                     self.set_ranges,
                                     list(range(self.dimensions)), copy.deepcopy(self.state.ranges_viewport), copy.deepcopy(self.state.range_level_show),
                                     axis_indices, ranges_viewport=self.state.ranges_viewport, range_level_show=range_level)
            self.checkUndoRedo()
        logger.debug("set axis/self.state.ranges_viewport: %r / %r" % (axis_indices, self.state.ranges_viewport))
        if axis_indices is None:  # signals a 'reset'
            for axis_index in range(self.dimensions):
                self.state.ranges_viewport[axis_index] = None
                for layer in self.layers:
                    layer.state.ranges_grid[axis_index] = None
                # self.ranges[axis_index] = None
        else:
            for i, axis_index in enumerate(axis_indices):
                if self.state.ranges_viewport:
                    self.state.ranges_viewport[axis_index] = ranges_viewport[i]
                    for layer in self.layers:
                        # layer.state.ranges_grid[axis_index] = self.state.ranges_viewport[i]
                        if self.state.ranges_viewport[i] is not None:
                            layer.set_range(self.state.ranges_viewport[i][0], ranges_viewport[i][1], axis_index)
                        else:
                            layer.set_range(None, None, axis_index)
                i  # f ranges:
                #   self.ranges[axis_index] = ranges[i]
        logger.debug("set range_level: %r" % (range_level, ))
        self.state.range_level_show = range_level
        if len(axis_indices) > 0:
            self.check_aspect(axis_indices[0])  # maybe we should use the widest or smallest one
        self.update_all_layers()
        # self.update_plot()

    def update_plot(self):
        # default value
        self.update_direct()

    def update_direct(self, layer=None):
        logger.info("update plot: ranges_viewport=%r" % (self.state.ranges_viewport, ))
        if layer:
            logger.info("only update layer %r (index %d)" % (layer, self.layers.index(layer)))
            layers = [layer]
        else:
            logger.info("updating all layers")
            layers = self.layers
        layers = [layer for layer in self.layers if layer.get_needs_update()]
        if not layers:
            logger.error("update requested while no layer needs it")

        timelog("begin computation", reset=True)
        # this can be
        promises = [layer.add_tasks_ranges() for layer in layers]
        executors = list(set([layer.dataset.executor for layer in layers]))
        # execute may do things delay, like at a server
        try:
            for executor in executors:
                executor.execute()
        except SyntaxError as e:
            msg = "%s: %r" % (e.args[0], e.args[1][3])
            qt.dialog_error(self, "Syntax error", "Syntax error: %s " % msg)
        except KeyError as e:
            msg = e.args[0]
            qt.dialog_error(self, "Unknown variable", "Unknown variable or column: %s " % msg)

        promise_ranges_done = vaex.promise.listPromise(promises)
        promise_ranges_done.then(self._update_step2, self.on_error_or_cancel).end()
        logger.debug("waiting for promises %r to finish", promises)

    def on_error_or_cancel(self, error):
        logger.exception("error occured: %r", error, exc_info=error)
        traceback.print_exc()
        # raise exception
        raise error

    def _update_step2(self, layers):
        """Each layer has it's own ranges_grid computed now, unless something went wrong
        But all layers are shown with the same ranges (self.state.ranges_viewport)
        If any of the ranges is None, take the min/max of each layer
        """
        logger.info("done with ranges, now update step2 for layers: %r", layers)

        for dimension in range(self.dimensions):
            if self.state.ranges_viewport[dimension] is None:
                vmin = min([layer.state.ranges_grid[dimension][0] for layer in layers])
                vmax = max([layer.state.ranges_grid[dimension][1] for layer in layers])
                self.state.ranges_viewport[dimension] = [vmin, vmax]

        logger.debug("ranges before aspect check: %r", self.state.ranges_viewport)
        self.check_aspect(0)
        logger.debug("ranges after aspect check: %r", self.state.ranges_viewport)
        # now make sure the layers all have the same ranges_grid
        for layer in layers:
            # layer.state.ranges_grid = copy.deepcopy(self.state.ranges_viewport)
            for d in range(layer.dimensions):
                layer.set_range(self.state.ranges_viewport[d][0], self.state.ranges_viewport[d][1], d)

        # now we are ready to calculate histograms
        promises = [layer.add_tasks_histograms() for layer in layers]
        executors = list(set([layer.dataset.executor for layer in layers]))
        for executor in executors:
            executor.execute()

        promises_histograms_done = vaex.promise.listPromise(promises)
        promises_histograms_done.then(self._update_step3, self.on_error_or_cancel).end()

    def _update_step3(self, layers):
        logger.info("done with histograms, now update step3, layers = %r" % layers)
        # all histograms are computed, and anything needed to visualize it
        # for 1d histograms, we for instance want to have similar levels
        if self.state.range_level_show is None:
            self.calculate_range_level_show()
        timelog("computation done")

        # now we can do the plot
        self.push_full_state()
        self.queue_replot()

    def queue_history_change(self, reason):
        self.full_state_history_change_reason = reason

    def get_full_state(self):
        return {"window": copy.deepcopy(dict(self.state)), "layers": [copy.deepcopy(dict(layer.state)) for layer in self.layers]}

    def push_full_state(self):
        if not self.full_state_history:
            self.full_state_history.append(("initial state", self.get_full_state()))
            self.full_state_history_index += 1
        else:
            if self.full_state_history_change_reason is not None:
                # clip off any possible redo history
                self.full_state_history = self.full_state_history[:self.full_state_history_index + 1]
                state = self.full_state_history.append((self.full_state_history_change_reason, self.get_full_state()))
                self.full_state_history_index += 1
                self.full_state_history_change_reason = None
        self.checkUndoRedo()

    def queue_push_full_state(self):
        QtCore.QTimer.singleShot(1, self.push_full_state)

    def restore_full_state(self, state):
        # return {"window":copy.deepcopy(dict(self.state)), "layers":[copy.deepcopy(dict(layer.state)) for layer in self.layers]}
        layer_states = [AttrDict(copy.deepcopy(layer)) for layer in state["layers"]]
        logger.debug("removing possible layers")
        while len(layer_states) < len(self.layers):
            logger.debug("removing layer: %r", self.layers[len(self.layers) - 1])
            self.remove_layer(layer_index=len(self.layers) - 1)
        for layer_state, layer in zip(layer_states, self.layers):
            logger.debug("restoring state of layer %r to %r", layer, layer_state)
            layer.restore_state(layer_state)
        leftover_layer_states = layer_states[len(self.layers):]
        for layer_state in leftover_layer_states:
            layer = self.add_layer(expressions=layer_state.expressions, dataset=self.datasets[layer_state.dataset_path], name=layer_state.name)
            logger.debug("restoring (left over) state of layer %r to %r", layer, layer_state)
            layer.restore_state(layer_state)
        logger.debug("queue history change")
        self.queue_history_change(None)
        # self.queue_update()
        self.state = AttrDict(copy.deepcopy(state["window"]))
        logger.debug("updating gui")
        self.action_axes_lock.setChecked(bool(self.state.axis_lock))
        self.action_aspect_lock_one.setChecked(bool(self.state.aspect))
        self.action_resolution_list[grid_resolutions.index(self.state.grid_size)].setChecked(True)
        self.action_resolution_vector_list[vector_grid_resolutions.index(self.state.vector_grid_size)].setChecked(True)
        logger.debug("update all layers")
        self.update_all_layers()
        # for layer in

    def calculate_range_level_show(self):
        layers = [layer for layer in self.layers if layer.range_level is not None]
        if layers:
            for layer in layers:
                logger.debug("layer %r has range_level %r" % (layer, layer.range_level))
            vmin = min([layer.range_level[0] for layer in layers])
            vmax = max([layer.range_level[1] for layer in layers])
            self.state.range_level_show = [vmin, vmax]
        logger.debug("range_level_show = %r" % (self.state.range_level_show, ))

    def set_range(self, min, max, dimension=0):
        was_equal = self.state.ranges_viewport[dimension] is not None and \
            list(self.state.ranges_viewport[dimension]) == [min, max]
        dimension_names = "xyz"
        dim_name = dimension_names[dimension]
        action = undo.ActionZoom(self.undoManager, "change range in dimension %s" % dim_name,
                                 self.set_ranges,
                                 list(range(self.dimensions)), copy.deepcopy(self.state.ranges_viewport), copy.deepcopy(self.state.range_level_show),
                                 [dimension], ranges_viewport=[[min, max]])
        action.do()
        self.checkUndoRedo()
        return was_equal

    def zoom(self, factor, axes, x=None, y=None, delay=300, *args):
        logger.info("zooming at location %r, with a factor of %r", (x, y), factor)
        if self.last_ranges_viewport is None:
            self.last_ranges_viewport = copy.deepcopy(self.state.ranges_viewport)
        if self.last_range_level_show is None:
            self.last_range_level_show = copy.deepcopy(self.state.range_level_show)
        xmin, xmax = axes.get_xlim()
        width = xmax - xmin

        if x is None:
            x = xmin + width / 2

        fraction = (x - xmin) / width

        range_level_show = None
        ranges_viewport = []
        ranges = []
        axis_indices = []

        ranges_viewport.append([x - width * fraction * factor, x + width * (1 - fraction) * factor])
        axis_indices.append(axes.xaxis_index)

        ymin, ymax = axes.get_ylim()
        height = ymax - ymin
        if y is None:
            y = ymin + height / 2
        fraction = (y - ymin) / height
        ymin_show, ymax_show = y - height * fraction * factor, y + height * (1 - fraction) * factor
        # ymin_show, ymax_show = min(ymin_show, ymax_show), max(ymin_show, ymax_show)
        if len(self.state.ranges_viewport) == 1:  # if 1d, y refers to range_level
            range_level_show = ymin_show, ymax_show
            if (QtGui.QApplication.keyboardModifiers() == QtCore.Qt.AltModifier) or (QtGui.QApplication.keyboardModifiers() == QtCore.Qt.ControlModifier):
                range_level_show = ymin, ymax
            else:
                range_level_show = ymin_show, ymax_show
        else:
            ranges_viewport.append([ymin_show, ymax_show])
            axis_indices.append(axes.yaxis_index)

        reason = "zoom " + ("out" if factor > 1 else "in")
        self.queue_history_change(reason)

        def delayed_zoom():
            action = undo.ActionZoom(self.undoManager, reason,
                                     self.set_ranges,
                                     list(range(self.dimensions)), self.last_ranges_viewport, self.last_range_level_show,
                                     axis_indices, ranges_viewport=ranges_viewport, range_level_show=range_level_show)
            self.last_ranges_viewport = None
            self.last_range_level_show = None
            action.do()
            self.checkUndoRedo()
        for layer in self.layers:
            layer.flag_needs_update()
        self.queue_update(delayed_zoom, delay=delay)

        if 1:
            self.check_aspect(axes.xaxis_index)
            if self.dimensions in [2, 3]:
                self.state.ranges_viewport[axes.xaxis_index] = list(ranges_viewport[0])
                # self.state.ranges_viewport[axes.xaxis_index].sort()
                self.state.ranges_viewport[axes.yaxis_index] = list(ranges_viewport[1])
                # self.state.ranges_viewport[axes.yaxis_index].sort()
                first_layer = None
                if len(self.layers):
                    first_layer = self.layers[0]
                for ax in self.getAxesList():
                    ax.set_xlim(self.state.ranges_viewport[ax.xaxis_index])
                    ax.set_ylim(self.state.ranges_viewport[ax.yaxis_index])
                    # if 1:
                    #   if first_layer.flip_x:
                    #       ax.invert_xaxis()
                    #   if first_layer.flip_y:
                    #       ax.invert_yaxis()
            if self.dimensions == 1:
                self.state.ranges_viewport[axis_indices[0]] = list(ranges_viewport[0])
                self.state.range_level_show = list(range_level_show)
                axes.set_xlim(self.state.ranges_viewport[0])
                axes.set_ylim(self.state.range_level_show)
            self.queue_redraw()
            # self.plot()

    def onActionSaveFigure(self, *ignore_args):
        filetypes = dict(self.fig.canvas.get_supported_filetypes())  # copy, otherwise we lose png support :)
        pngtype = [("png", filetypes["png"])]
        del filetypes["png"]
        filetypes = [value + "(*.%s)" % key for (key, value) in pngtype + list(filetypes.items())]
        import string

        def make_save(expr):
            save_expr = ""
            for char in expr:
                if char not in string.whitespace:
                    if char in string.ascii_letters or char in string.digits or char in "._":
                        save_expr += char
                    else:
                        save_expr += "_"
            return save_expr
        layer = self.current_layer
        if layer is not None:
            save_expressions = list(map(make_save, layer.state.expressions))
            type = "histogram" if self.dimensions == 1 else "density"
            filename = layer.dataset.name + "_%s_" % type + "-vs-".join(save_expressions) + ".png"
            filename = QtGui.QFileDialog.getSaveFileName(self, "Export to figure", filename, ";;".join(filetypes))
            if isinstance(filename, tuple):
                filename = filename[0]
            filename = str(filename)
            if filename:
                logger.debug("saving to figure: %s" % filename)
                self.fig.savefig(filename)
                self.filename_figure_last = filename
                self.action_save_figure_again.setEnabled(True)

    def onActionSaveFigureAgain(self, *ignore_args):
        logger.debug("saving to figure: %s" % self.filename_figure_last)
        self.fig.savefig(self.filename_figure_last)

    def get_aspect(self):
        if 0:
            xmin, xmax = self.axes.get_xlim()
            ymin, ymax = self.axes.get_ylim()
            height = ymax - ymin
            width = xmax - xmin
        return 1  # width/height

    def onActionAspectLockOne(self, *ignore_args):
        self.state.aspect = self.get_aspect() if self.action_aspect_lock_one.isChecked() else None
        logger.debug("set aspect to: %r" % self.state.aspect)
        self.check_aspect(0)
        self.queue_history_change("Set equal aspect" if self.state.aspect else "Unset equal aspect")
        self.update_all_layers()
        # self.compute()
        # self.dataset.executor.execute()
        # self.plot()

    def _onActionAspectLockOne(self, *ignore_args):
        self.state.aspect = 1  # self.get_aspect() if self.action_aspect_lock.isEnabled() else None
        logger.debug("set aspect to: %r" % self.state.aspect)

    def onActionExport(self):
        if self.dimensions == 3:
            ok, mask = select_many(self, "Choose export options", ["Export grid", "Export grid from selection", "Export vector grid"])
            name = self.current_layer.dataset.name
            name = gettext(self, "Export name", "Give a base name for the export files", name)
            if name:
                dir_path = getdir(self, "Choose directory where to save")
                if dir_path is None:
                    return
                msg_list = []
                yesall = False

                optionsname = os.path.join(dir_path, name + "_meta.json")
                options = {}
                options["extent"] = list(self.current_layer.state.ranges_grid[0]) + list(self.current_layer.state.ranges_grid[1])
                with open(optionsname, "w") as f:
                    json.dump(options, f, indent=4)
                msg_list.append("wrote: " + optionsname)

                if mask[0]:
                    gridname = os.path.join(dir_path, name + "_grid.npy")
                    if not os.path.exists(gridname) or yesall:
                        yes, yesall = True, yesall
                    else:
                        yes, yesall = dialog_confirm(self, "Overwrite", "Overwrite: " + gridname, to_all=True)
                    if yes or yesall:
                        np.save(gridname, self.current_layer.amplitude_grid)
                        msg_list.append("wrote: " + gridname)
                if mask[1]:
                    if self.current_layer.dataset.mask is not None:
                        gridname = os.path.join(dir_path, name + "_grid_selection.npy")
                        if not os.path.exists(gridname) or yesall:
                            yes, yesall = True, yesall
                        else:
                            yes, yesall = dialog_confirm(self, "Overwrite", "Overwrite: " + gridname, to_all=True)
                        if yes or yesall:
                            np.save(gridname, self.current_layer.amplitude_selection)
                            msg_list.append("wrote: " + gridname)
                if mask[2]:
                    if hasattr(self.current_layer, "vector_grids"):
                        gridname = os.path.join(dir_path, name + "_grid_vector.npy")
                        if not os.path.exists(gridname) or yesall:
                            yes, yesall = True, yesall
                        else:
                            yes, yesall = dialog_confirm(self, "Overwrite", "Overwrite: " + gridname, to_all=True)
                        if yes or yesall:
                            np.save(gridname, self.current_layer.vector_grids)
                            msg_list.append("wrote: " + gridname)
                msg = "\n".join(msg_list)
                dialog_info(self, "Finished export", msg)
        elif self.dimensions == 2:
            ok, mask = select_many(self, "Choose export options", ["Export matplotlib python script", "Export grid", "Export selection", "Export vector grid"])
            if ok:
                name = self.current_layer.dataset.name
                name = gettext(self, "Export name", "Give a base name for the export files", name)
                if name:
                    dir_path = getdir(self, "Choose directory where to save")
                    if dir_path is None:
                        return
                    msg_list = []

                    yesall = False
                    if mask[0]:
                        scriptname = os.path.join(dir_path, name + "_plot.py")
                        template = vaex.ui.templates.matplotlib
                        if not os.path.exists(scriptname):
                            yes, yesall = True, False
                        else:
                            yes, yesall = dialog_confirm(self, "Overwrite", "Overwrite: " + scriptname, to_all=True)
                        if yes or yesall:
                            with open(scriptname, "w") as f:
                                f.write(template.format(name=name))
                                msg_list.append("wrote: " + scriptname)

                    optionsname = os.path.join(dir_path, name + "_meta.json")
                    options = {}
                    options["extent"] = list(self.current_layer.state.ranges_grid[0]) + list(self.current_layer.state.ranges_grid[1])
                    with open(optionsname, "w") as f:
                        json.dump(options, f, indent=4)
                    msg_list.append("wrote: " + optionsname)

                    if mask[1]:
                        gridname = os.path.join(dir_path, name + "_grid.npy")
                        if not os.path.exists(gridname) or yesall:
                            yes, yesall = True, yesall
                        else:
                            yes, yesall = dialog_confirm(self, "Overwrite", "Overwrite: " + gridname, to_all=True)
                        if yes or yesall:
                            np.save(gridname, self.current_layer.amplitude_grid)
                            msg_list.append("wrote: " + gridname)
                        if mask[2]:
                            if self.current_layer.dataset.mask is not None:
                                gridname = os.path.join(dir_path, name + "_grid_selection.npy")
                                if not os.path.exists(gridname) or yesall:
                                    yes, yesall = True, yesall
                                else:
                                    yes, yesall = dialog_confirm(self, "Overwrite", "Overwrite: " + gridname, to_all=True)
                                if yes or yesall:
                                    np.save(gridname, self.current_layer.amplitude_selection)
                                    msg_list.append("wrote: " + gridname)
                        if mask[3]:
                            if hasattr(self.current_layer, "vector_grids"):
                                gridname = os.path.join(dir_path, name + "_grid_vector.npy")
                                if not os.path.exists(gridname) or yesall:
                                    yes, yesall = True, yesall
                                else:
                                    yes, yesall = dialog_confirm(self, "Overwrite", "Overwrite: " + gridname, to_all=True)
                                if yes or yesall:
                                    np.save(gridname, self.current_layer.vector_grids)
                                    msg_list.append("wrote: " + gridname)
                        msg = "\n".join(msg_list)
                        dialog_info(self, "Finished export", msg)
        else:
            dialog_info(self, "Export failure", "Oops sorry, export only supported for 2d and 3d (partially) plots at the moment")

    def active_layers(self):
        return [self.current_layer] if self.current_layer else self.layers

    def update_favorite_selections(self):
        # pass
        # self.menu_selection.clear()
        if hasattr(self, "_favorite_selections_actions"):
            for action in self._favorite_selections_actions:
                self.menu_selection.removeAction(action)
        self._favorite_selections_actions = []  # keep references
        if self.current_layer:
            for key, value in self.current_layer.dataset.favorite_selections.items():
                def select(ignore=None, key=key):
                    self.current_layer.dataset.selection_favorite_apply(name=key)
                action = QtGui.QAction(key, self)
                action.triggered.connect(select)
                self._favorite_selections_actions.append(action)
                self.menu_selection.addAction(action)

    def addToolbar2(self, layout, contrast=True, gamma=True):
        self.toolbar2 = QtGui.QToolBar(self)
        self.toolbar2.setToolButtonStyle(QtCore.Qt.ToolButtonTextUnderIcon)
        self.toolbar2.setIconSize(QtCore.QSize(16, 16))

        layout.addWidget(self.toolbar2)

        def on_store_selection():
            if self.current_layer:
                dataset = self.current_layer.dataset
                if not dataset.has_selection():
                    dialog_error(self, "No selection", "No selection made")
                else:
                    path = dataset.name + "-selection.yaml"
                    path = dialogs.get_path_save(self, "Save selection", path, "YAML (*.yaml);;JSON (*.json)")
                    if path:
                        _, ext = os.path.splitext(path)
                        data = dataset.get_selection().to_dict()
                        if ext == ".yaml":
                            import yaml
                            with open(path, "w") as f:
                                yaml.dump(data, f)
                        elif ext == ".json":
                            import json
                            with open(path, "w") as f:
                                json.dump(data, f)
                        else:
                            dialogs.dialog_error(self, "Unknown extension", "Unknown extension %r" % ext)
                        # np.save(path, self.dataset.mask)

        self.action_selection_store = QtGui.QAction(QtGui.QIcon(iconfile('tag-export')), '&Export selection query', self)
        self.action_selection_store.triggered.connect(on_store_selection)
        # self.action_selection_store.setCheckable(True)
        # self.toolbar2.addAction(self.action_selection_store)
        self.menu_selection.addAction(self.action_selection_store)

        def on_load_selection():
            if self.current_layer:
                dataset = self.current_layer.dataset
                path = dataset.name + "-selection.yaml"
                path = get_path_open(self, "Open selection", path, "Compatible files for vaex (*.yaml *.json)")
                if path:
                    _, ext = os.path.splitext(path)
                    if ext == ".yaml":
                        import yaml
                        with open(path) as f:
                            data = yaml.load(f)
                    elif ext == ".json":
                        import json
                        with open(path) as f:
                            json.load(f)
                    else:
                        dialogs.dialog_error(self, "Unknown extension", "Unknown extension %r" % ext)
                    try:
                        selection = vaex.dataset.selection_from_dict(dataset, data)
                    except Exception as e:
                        logger.exception("error reading in selection")
                        dialogs.dialog_error(self, "Error reading in selection", "Error reading in selection: %r" % e)
                    dataset.set_selection(selection)
                    self.queue_update()
        self.action_selection_load = QtGui.QAction(QtGui.QIcon(iconfile('tag-import')), '&Import selection query', self)
        self.action_selection_load.triggered.connect(on_load_selection)
        # self.action_selection_load.setCheckable(True)
        # self.toolbar2.addAction(self.action_selection_load)
        self.menu_selection.addAction(self.action_selection_load)

        self.menu_selection.addSeparator()

        def on_selection_copy():
            if self.current_layer:
                selection = self.current_layer.dataset.get_selection()
                if self.current_layer:
                    if selection:
                        data = selection.to_dict()
                        clipboard = QtGui.QApplication.clipboard()
                        f = StringIO()
                        vaex.utils.yaml_dump(f, data)
                        text = str(f.getvalue())
                        clipboard.setText(text)
                    else:
                        dialogs.dialog_error(self, "No selection", "No selection exists")
            else:
                dialogs.dialog_error(self, "No layer", "No active layer")

        self.action_selection_copy = QtGui.QAction('&Copy selection query', self)
        self.action_selection_copy.triggered.connect(on_selection_copy)
        # self.action_selection_load.setCheckable(True)
        # self.toolbar2.addAction(self.action_selection_load)
        self.menu_selection.addAction(self.action_selection_copy)

        def on_selection_paste():
            clipboard = QtGui.QApplication.clipboard()
            f = StringIO()
            f.write(clipboard.text())
            f.seek(0)
            data = vaex.utils.yaml_load(f)
            for layer in self.active_layers():
                selection = vaex.dataset.selection_from_dict(layer.dataset, data)
                layer.dataset.set_selection(selection)

            #   dialogs.dialog_error(self, "No selection", "No selection exists")

        self.action_selection_paste = QtGui.QAction('&Paste selection query', self)
        self.action_selection_paste.triggered.connect(on_selection_paste)
        # self.action_selection_load.setCheckable(True)
        # self.toolbar2.addAction(self.action_selection_load)
        self.menu_selection.addAction(self.action_selection_paste)

        self.menu_selection.addSeparator()

        def on_selection_add_favorite():
            if self.current_layer:
                # data = self.current_layer.dataset.get_selection().to_dict()
                dataset = self.current_layer.dataset
                if dataset.get_selection():
                    name = dialogs.gettext(self, "Add selection to favorites", "Name", "my selection")
                    if name:
                        if name not in dataset.favorite_selections or dialogs.dialog_confirm(self, "Overwrite", "Overwrite selection with the same name?"):
                            dataset.selection_favorite_add(name)
                            self.update_favorite_selections()
                else:
                    dialogs.dialog_error(self, "No selection", "No selection exists")
            else:
                dialogs.dialog_error(self, "No layer", "No active layer")

        self.action_selection_add_favorites = QtGui.QAction(QtGui.QIcon(iconfile('star')), '&Add to favorites', self)
        self.action_selection_add_favorites.triggered.connect(on_selection_add_favorite)
        self.menu_selection.addAction(self.action_selection_add_favorites)

        def on_selection_remove_favorite():
            if self.current_layer:
                # data = self.current_layer.dataset.get_selection().to_dict()
                dataset = self.current_layer.dataset
                if dataset.favorite_selections:
                    names = list(dataset.favorite_selections.keys())
                    index = dialogs.choose(self, "Remove favorite", "Favorite", names)
                    if index is not None:
                        key = names[index]
                        dataset.selection_favorite_remove(key)
                        self.update_favorite_selections()
                else:
                    dialogs.dialog_error(self, "No favorites", "No favorites exists")
            else:
                dialogs.dialog_error(self, "No layer", "No active layer")

        self.action_selection_remove_favorites = QtGui.QAction(QtGui.QIcon(iconfile('star--minus')), '&Remove a favorite', self)
        self.action_selection_remove_favorites.triggered.connect(on_selection_remove_favorite)
        self.menu_selection.addAction(self.action_selection_remove_favorites)

        self.action_save_figure = QtGui.QAction(QtGui.QIcon(iconfile('image-export')), '&Export figure', self)
        self.action_save_figure_again = QtGui.QAction(QtGui.QIcon(iconfile('image-export')), '&Export figure again', self)
        # self.menu_save = QtGui.QMenu(self)
        # self.action_save_figure.setMenu(self.menu_save)
        # self.menu_save.addAction(self.action_save_figure_again)
        # self.toolbar2.addAction(self.action_save_figure)
        self.menu_file.addAction(self.action_save_figure)
        self.menu_file.addAction(self.action_save_figure_again)

        self.action_save_figure.triggered.connect(self.onActionSaveFigure)
        self.action_save_figure_again.triggered.connect(self.onActionSaveFigureAgain)
        self.action_save_figure_again.setEnabled(False)

        self.action_export = QtGui.QAction(QtGui.QIcon(iconfile('script-export')), '&Export data/script', self)
        self.menu_file.addSeparator()
        self.menu_file.addAction(self.action_export)
        self.action_export.triggered.connect(self.onActionExport)

        self.action_aspect_lock_one = QtGui.QAction(QtGui.QIcon(iconfile('control-stop-square')), 'Equal aspect', self)
        self.action_aspect_lock_one.setShortcut("Ctrl+=")
        # self.action_aspect_lock_one = QtGui.QAction(QtGui.QIcon(iconfile('table_save')), '&Set aspect to one', self)
        # self.menu_aspect = QtGui.QMenu(self)
        # self.action_aspect_lock.setMenu(self.menu_aspect)
        # self.menu_aspect.addAction(self.action_aspect_lock_one)
        # self.toolbar2.addAction(self.action_aspect_lock_one)
        self.menu_view.insertAction(self.action_mini_mode_normal, self.action_aspect_lock_one)

        # self.action_aspect_lock.triggered.connect(self.onActionAspectLock)
        self.action_aspect_lock_one.setCheckable(True)
        self.action_aspect_lock_one.triggered.connect(self.onActionAspectLockOne)
        # self.action_save_figure_again.setEnabled(False)

        self.action_undo = QtGui.QAction(QtGui.QIcon(iconfile('arrow-curve-180-left')), 'Back', self)
        self.action_redo = QtGui.QAction(QtGui.QIcon(iconfile('arrow-curve-000-left')), 'Forward', self)
        self.action_undo.setShortcut("Ctrl+Left")
        self.action_redo.setShortcut("Ctrl+Right")

        self.toolbar.insertAction(self.action_move, self.action_undo)
        self.toolbar.insertAction(self.action_move, self.action_redo)
        # self.toolbar2.addAction(self.action_undo)
        # self.toolbar2.addAction(self.action_redo)
        self.menu_view.insertAction(self.action_aspect_lock_one, self.action_undo)
        self.menu_view.insertAction(self.action_aspect_lock_one, self.action_redo)
        self.action_undo.triggered.connect(self.onActionUndo)
        self.action_redo.triggered.connect(self.onActionRedo)
        self.checkUndoRedo()

        self.action_disjoin = QtGui.QAction(QtGui.QIcon(iconfile('sql-join-outer-exclude')), 'Disjoined', self)
        self.action_disjoin.setCheckable(True)
        self.action_disjoin.triggered.connect(self.onActionDisjoin)
        self.action_disjoin.setShortcut("Ctrl+D")
        self.menu_view.insertAction(self.action_mini_mode_normal, self.action_disjoin)
        # self.toolbar2.addAction(self.action_disjoin)

        self.action_axes_lock = QtGui.QAction(QtGui.QIcon(iconfile('lock')), 'Lock axis', self)
        self.action_axes_lock.setCheckable(True)
        self.action_axes_lock.triggered.connect(self.onActionAxesLock)
        self.action_axes_lock.setShortcut("Ctrl+Shift+L")
        self.menu_view.insertAction(self.action_mini_mode_normal, self.action_axes_lock)
        # self.toolbar2.addAction(self.action_axes_lock)

        self.menu_view.insertSeparator(self.action_aspect_lock_one)
        self.menu_view.insertSeparator(self.action_mini_mode_normal)
        self.toolbar2.setVisible(False)

    def onActionAxesLock(self, ignore=None):
        self.state.axis_lock = self.action_axes_lock.isChecked()
        self.queue_history_change("Set axis lock" if self.action_axes_lock.isChecked() else "Unset axis lock")
        self.push_full_state()
        self.checkUndoRedo()

    def onActionShuffled(self, ignore=None):
        self.xoffset = 1 if self.action_shuffled.isChecked() else 0
        self.compute()
        self.dataset.executor.execute()
        logger.debug("xoffset = %r" % self.xoffset)

    def onActionDisjoin(self, ignore=None):
        # self.xoffset = 1 if self.action_shuffled.isChecked() else 0
        layer = self.current_layer
        if layer:
            layer.show_disjoined = self.action_disjoin.isChecked()
            layer.calculate_amplitudes()
            logger.debug("show_disjoined = %r" % layer.show_disjoined)
            self.queue_replot()

    def addToolbar(self, layout, pick=True, xselect=True, yselect=True, lasso=True):

        self.toolbar = QtGui.QToolBar(self)
        self.toolbar.setToolButtonStyle(QtCore.Qt.ToolButtonTextUnderIcon)
        self.action_group_main = QtGui.QActionGroup(self)
        self.action_group_mainSelectMode = QtGui.QActionGroup(self)

        self.action_group_display = QtGui.QActionGroup(self)

        self.actiongroup_display_mode = QtGui.QActionGroup(self)

        # self.action_displmini_mode = QtGui.QAction(QtGui.QIcon(iconfile('picture_empty')), '&Mini screen(should not see)', self)
        self.action_mini_mode_normal = QtGui.QAction(QtGui.QIcon(iconfile('layout-hf-2')), 'Normal', self)
        self.action_mini_mode_compact = QtGui.QAction(QtGui.QIcon(iconfile('layout-hf')), 'Compact', self)
        self.action_mini_mode_ultra = QtGui.QAction(QtGui.QIcon(iconfile('layout')), 'Ultra compact', self)
        self.action_mini_mode_normal.setShortcut("Ctrl+Shift+N")
        self.action_mini_mode_compact.setShortcut("Ctrl+Shift+C")
        self.action_mini_mode_ultra.setShortcut("Ctrl+Shift+U")

        self.action_group_mini_mode = QtGui.QActionGroup(self)
        self.action_group_mini_mode.addAction(self.action_mini_mode_normal)
        self.action_group_mini_mode.addAction(self.action_mini_mode_compact)
        self.action_group_mini_mode.addAction(self.action_mini_mode_ultra)

        self.action_fullscreen = QtGui.QAction(QtGui.QIcon(iconfile('picture_empty')), '&Fullscreen', self)
        self.action_fullscreen.setCheckable(True)
        self.action_fullscreen.setShortcut(("Ctrl+F"))

        self.action_toolbar_toggle = QtGui.QAction(QtGui.QIcon(iconfile('ui-toolbar')), '&Toolbars', self)
        self.action_toolbar_toggle.setCheckable(True)
        self.action_toolbar_toggle.setChecked(True)
        self.action_toolbar_toggle.setShortcut(("Ctrl+Shift+T"))

        # self.actiongroup_mini_mode.addAction(self.action_fullscreen)

        def toggle_fullscreen(ignore=None):
            fullscreen = self.windowState() & QtCore.Qt.WindowFullScreen
            fullscreen = not fullscreen  # toggle
            self.action_fullscreen.setChecked(fullscreen)
            if fullscreen:
                self.setWindowState(self.windowState() | QtCore.Qt.WindowFullScreen)
            else:
                self.setWindowState(self.windowState() ^ QtCore.Qt.WindowFullScreen)

        self.action_fullscreen.triggered.connect(toggle_fullscreen)

        self.action_toolbar_toggle.triggered.connect(self.on_toolbar_toggle)

        self.action_move = QtGui.QAction(QtGui.QIcon(iconfile('edit-move')), '&Move', self)
        self.action_move.setShortcut("Ctrl+M")
        self.menu_mode.addAction(self.action_move)

        self.action_pick = QtGui.QAction(QtGui.QIcon(iconfile('cursor')), '&Pick', self)
        self.action_pick.setShortcut("Ctrl+P")
        self.menu_mode.addAction(self.action_pick)

        self.action_select = QtGui.QAction(QtGui.QIcon(iconfile('glue_lasso16')), '&Select(you should not read this)', self)
        self.action_xrange = QtGui.QAction(QtGui.QIcon(iconfile('glue_xrange_select16')), '&x-range', self)
        self.action_yrange = QtGui.QAction(QtGui.QIcon(iconfile('glue_yrange_select16')), '&y-range', self)
        self.action_lasso = QtGui.QAction(QtGui.QIcon(iconfile('glue_lasso16')), '&Lasso', self)
        self.action_select_rectangle = QtGui.QAction(QtGui.QIcon(iconfile('glue_square16')), '&Rectangle', self)
        self.action_slice = QtGui.QAction(QtGui.QIcon(iconfile('cutlery-knife')), '&Slice', self)
        self.action_select_viewport = QtGui.QAction(QtGui.QIcon(iconfile('control-stop-square')), '&Select viewport', self)
        self.action_select_none = QtGui.QAction(QtGui.QIcon(iconfile('cross')), '&No selection', self)
        self.action_select_invert = QtGui.QAction(QtGui.QIcon(iconfile('arrow-circle-double-135')), '&Invert', self)

        self.action_xrange.setShortcut("Ctrl+Shift+X")
        self.menu_mode.addAction(self.action_xrange)
        self.menu_selection.addAction(self.action_xrange)
        self.action_yrange.setShortcut("Ctrl+Shift+Y")
        self.menu_mode.addAction(self.action_yrange)
        self.menu_selection.addAction(self.action_yrange)
        self.action_lasso.setShortcut("Ctrl+L")
        self.menu_mode.addAction(self.action_lasso)
        self.menu_selection.addAction(self.action_lasso)
        self.action_select_rectangle.setShortcut("Ctrl+R")
        self.menu_mode.addAction(self.action_select_rectangle)
        self.menu_selection.addAction(self.action_select_rectangle)

        # self.action_slice.setShortcut("Ctrl+S")
        # self.menu_mode.addAction(self.action_slice)

        self.menu_selection.addSeparator()

        self.action_select_viewport.setShortcut("Ctrl+Shift+V")
        self.menu_selection.addAction(self.action_select_viewport)
        self.action_select_none.setShortcut("Ctrl+N")
        self.menu_selection.addAction(self.action_select_none)
        self.action_select_invert.setShortcut("Ctrl+I")
        self.menu_selection.addAction(self.action_select_invert)
        self.menu_selection.addSeparator()

        self.menu_mode.addSeparator()
        self.action_select_invert.setShortcut("Ctrl+I")
        # self.menu_mode_select_mode = QtGui.QMenu("Select mode")
        # self.menu_mode.addMenu(self.menu_mode_select_mode)

        self.action_select_mode_replace = QtGui.QAction(QtGui.QIcon(iconfile('sql-join-right')), '&Replace', self)
        self.action_select_mode_and = QtGui.QAction(QtGui.QIcon(iconfile('sql-join-inner')), '&And', self)
        self.action_select_mode_or = QtGui.QAction(QtGui.QIcon(iconfile('sql-join-outer')), '&Or', self)
        self.action_select_mode_xor = QtGui.QAction(QtGui.QIcon(iconfile('sql-join-outer-exclude')), 'Xor', self)
        self.action_select_mode_subtract = QtGui.QAction(QtGui.QIcon(iconfile('sql-join-left-exclude')), 'Subtract', self)
        self.action_select_mode_replace.setShortcut("Ctrl+Shift+=")
        self.action_select_mode_and.setShortcut("Ctrl+Shift+&")
        self.action_select_mode_or.setShortcut("Ctrl+Shift+|")
        self.action_select_mode_xor.setShortcut("Ctrl+Shift+^")
        self.action_select_mode_subtract.setShortcut("Ctrl+Shift+-")
        self.menu_mode.addAction(self.action_select_mode_replace)
        self.menu_mode.addAction(self.action_select_mode_and)
        self.menu_mode.addAction(self.action_select_mode_or)
        self.menu_mode.addAction(self.action_select_mode_xor)
        self.menu_mode.addAction(self.action_select_mode_subtract)

        self.action_samp_send_table_select_row_list = QtGui.QAction(QtGui.QIcon(iconfile('block--arrow')), 'Broadcast selection over SAMP', self)
        self.action_samp_send_table_select_row_list.setShortcut('Ctrl+Shift+B')
        # self.toolbar.addAction(self.action_samp_sand_table_select_row_list)
        self.menu_samp.addAction(self.action_samp_send_table_select_row_list)

        def send_samp_selection(ignore=None):
            self.signal_samp_send_selection.emit(self.dataset)
        self.send_samp_selection_reference = send_samp_selection  # does this fix the bug that clicking the buttons doesn't do anything?
        self.action_samp_send_table_select_row_list.triggered.connect(send_samp_selection)

        self.action_display_mode_both = QtGui.QAction(QtGui.QIcon(iconfile('picture_empty')), 'Show both', self)
        self.action_display_mode_full = QtGui.QAction(QtGui.QIcon(iconfile('picture_empty')), 'Show full', self)
        self.action_display_mode_selection = QtGui.QAction(QtGui.QIcon(iconfile('picture_empty')), 'Show selection', self)
        self.action_display_mode_both_contour = QtGui.QAction(QtGui.QIcon(iconfile('picture_empty')), 'Show contour', self)

        # self.actions_resolution = []

        self.actions_display = [self.action_display_mode_both, self.action_display_mode_full, self.action_display_mode_selection, self.action_display_mode_both_contour]
        for action in self.actions_display:
            self.action_group_display.addAction(action)
            action.setCheckable(True)
        action = self.actions_display[0]
        action.setChecked(True)
        self.action_display_current = action
        self.action_group_display.triggered.connect(self.onActionDisplay)
        # self.zoomButton = QtGui.QToolButton(self, )
        # $self.zoomButton.setIcon(QtGui.QIcon(iconfile('glue_zoom_to_rect')))
        # self.zoomMenu = QtGui.QMenu(self)
        # self.zoomMenu.addAction(self.action_zoom_x)
        # self.zoomMenu.addAction(self.action_zoom_y)
        # self.zoomMenu.addAction(self.action_zoom_out)
        # self.action_zoom.setMenu(self.zoomMenu)
        # self.action_zoom = self.toolbar.addWidget(self.zoomButton)

        # self.action_zoom = QtGui.QAction(QtGui.QIcon(iconfile('glue_zoom_to_rect')), '&Zoom', self)
        # exitAction.setShortcut('Ctrl+Q')
        # onExAction.setStatusTip('Exit application')

        # self.action_group_main.setToggleAction(True)
        # self.action_group_main.setExclusive(True)
        self.action_group_mainSelectMode.addAction(self.action_select_mode_replace)
        self.action_group_mainSelectMode.addAction(self.action_select_mode_and)
        self.action_group_mainSelectMode.addAction(self.action_select_mode_or)
        self.action_group_mainSelectMode.addAction(self.action_select_mode_xor)
        self.action_group_mainSelectMode.addAction(self.action_select_mode_subtract)

        self.action_group_main.addAction(self.action_move)
        self.action_group_main.addAction(self.action_pick)
        self.action_group_main.addAction(self.action_xrange)
        self.action_group_main.addAction(self.action_yrange)
        self.action_group_main.addAction(self.action_lasso)
        self.action_group_main.addAction(self.action_select_rectangle)
        if self.enable_slicing:
            self.action_group_main.addAction(self.action_slice)
        # self.action_group_main.addAction(self.action_zoom_out)

        self.menu_view.addAction(self.action_mini_mode_normal)
        self.menu_view.addAction(self.action_mini_mode_compact)
        self.menu_view.addAction(self.action_mini_mode_ultra)
        self.menu_view.addSeparator()
        self.menu_view.addAction(self.action_fullscreen)
        self.menu_view.addAction(self.action_toolbar_toggle)

        self.menu_view.addSeparator()
        self.action_group_resolution = QtGui.QActionGroup(self)
        self.action_resolution_list = []
        for index, resolution in enumerate(grid_resolutions):
            action_resolution = QtGui.QAction('Grid Resolution: %d' % resolution, self)

            def do(ignore=None, resolution=resolution):
                self.state.grid_size = resolution
                self.queue_history_change("Set grid resolution to %s" % resolution)
                self.update_all_layers()
            action_resolution.setCheckable(True)
            # TODO: this need to move to a layer change event
            if resolution == int(self.state.grid_size):
                action_resolution.setChecked(True)
            # action_resolution.setEnabled(False)
            action_resolution.triggered.connect(do)
            action_resolution.setShortcut("Ctrl+Alt+%d" % (index + 1))
            self.menu_view.addAction(action_resolution)
            self.action_group_resolution.addAction(action_resolution)
            self.action_resolution_list.append(action_resolution)

        self.menu_view.addSeparator()
        self.action_group_resolution_vector = QtGui.QActionGroup(self)
        self.action_resolution_vector_list = []
        for index, resolution in enumerate(vector_grid_resolutions):
            action_resolution = QtGui.QAction('Vector grid Resolution: %d' % resolution, self)

            def do(ignore=None, resolution=resolution):
                self.state.vector_grid_size = resolution
                self.queue_history_change("Set vector grid resolution to %s" % resolution)
                self.update_all_layers()
                # self.compute()
                # self.dataset.executor.execute()
            action_resolution.setCheckable(True)
            # TODO: this need to move to a layer change event
            if resolution == int(self.state.vector_grid_size):
                action_resolution.setChecked(True)
            # action_resolution.setEnabled(False)
            action_resolution.triggered.connect(do)
            action_resolution.setShortcut("Ctrl+Shift+Alt+%d" % (index + 1))
            self.menu_view.addAction(action_resolution)
            self.action_group_resolution_vector.addAction(action_resolution)
            self.action_resolution_vector_list.append(action_resolution)

        # self.mini_mode_button = QtGui.QToolButton()
        # self.mini_mode_button.setPopupMode(QtGui.QToolButton.InstantPopup)
        # self.mini_mode_button.setToolButtonStyle(QtCore.Qt.ToolButtonTextUnderIcon)
        # self.menu_mini_mode = QtGui.QMenu()
        # self.action_mini_mode.setMenu(self.menu_mini_mode)
        # self.mini_mode_button.setMenu(self.mini_mode_button_menu)
        # self.menu_mini_mode.addAction(self.action_mini_mode_normal)
        # self.menu_mini_mode.addAction(self.action_mini_mode_ultra)
        # self.mini_mode_button.setDefaultAction(self.action_miniscreen)
        # self.mini_mode_button.setCheckable(True)
        # self.mini_mode_button.setIcon(self.action_miniscreen.icon())
        # self.mini_mode_button.setText(self.action_miniscreen.text())

        # self.toolbar.addAction(self.action_mini_mode)
        # self.toolbar.addAction(self.action_fullscreen)
        # self.toolbar.addAction(self.action_toolbar_toggle)

        self.toolbar.addAction(self.action_move)
        if pick:
            self.toolbar.addAction(self.action_pick)
            # self.action_pick.setChecked(True)
            # self.setMode(self.action_pick, force=True)
            self.lastAction = self.action_pick
        self.toolbar.addAction(self.action_select)
        self.select_menu = QtGui.QMenu()
        self.action_select.setMenu(self.select_menu)
        self.select_menu.addAction(self.action_lasso)
        self.select_menu.addAction(self.action_select_rectangle)
        if yselect:
            # self.toolbar.addAction(self.action_yrange)
            self.select_menu.addAction(self.action_yrange)
            if self.dimensions > 1:
                self.lastActionSelect = self.action_yrange
        if xselect:
            # self.toolbar.addAction(self.action_xrange)
            self.select_menu.addAction(self.action_xrange)
            self.lastActionSelect = self.action_xrange
        if lasso:
            # self.toolbar.addAction(self.action_lasso)
            if self.dimensions > 1:
                self.lastActionSelect = self.action_lasso
        else:
            self.action_lasso.setEnabled(False)
            self.action_select_rectangle.setEnabled(False)
        if self.enable_slicing:
            self.toolbar.addAction(self.action_slice)
        self.select_menu.addSeparator()
        self.select_menu.addAction(self.action_select_viewport)
        self.select_menu.addAction(self.action_select_none)
        self.select_menu.addAction(self.action_select_invert)
        self.select_menu.addSeparator()

        self.select_mode_button = QtGui.QToolButton()
        self.select_mode_button.setPopupMode(QtGui.QToolButton.InstantPopup)
        self.select_mode_button.setToolButtonStyle(QtCore.Qt.ToolButtonTextUnderIcon)
        self.select_mode_button_menu = QtGui.QMenu()
        self.select_mode_button.setMenu(self.select_mode_button_menu)

        self.select_mode_button_menu.addAction(self.action_select_mode_replace)
        self.select_mode_button_menu.addAction(self.action_select_mode_or)
        self.select_mode_button_menu.addAction(self.action_select_mode_and)
        self.select_mode_button_menu.addAction(self.action_select_mode_xor)
        self.select_mode_button_menu.addAction(self.action_select_mode_subtract)
        self.select_mode_button.setDefaultAction(self.action_select_mode_replace)
        self.toolbar.addWidget(self.select_mode_button)

        # self.toolbar.addAction(action_select_mode)
        if 0:
            self.toolbar.addAction(self.action_zoom)
            self.zoom_menu = QtGui.QMenu()
            self.action_zoom.setMenu(self.zoom_menu)
            self.zoom_menu.addAction(self.action_zoom_rect)
            self.zoom_menu.addAction(self.action_zoom_x)
            self.zoom_menu.addAction(self.action_zoom_y)
            if self.dimensions == 1:
                self.lastActionZoom = self.action_zoom_x  # this makes more sense for histograms as default
            else:
                self.lastActionZoom = self.action_zoom_rect

            self.toolbar.addSeparator()
            # self.toolbar.addAction(self.action_zoom_out)
            self.toolbar.addAction(self.action_zoom_fit)
            # self.toolbar.addAction(self.action_zoom_use)
        else:
            # self.plugin_zoom.plug()
            #
            plugin_chain_toolbar = sorted(self.plugin_queue_toolbar, key=itemgetter(1))  # sort by order field
            for plug, order in plugin_chain_toolbar:
                plug()

        # self.zoomButton.setPopupMode(QtCore.QToolButton.DelayedPopup)

        self.action_group_main.triggered.connect(self.setMode)
        self.action_group_mainSelectMode.triggered.connect(self.setSelectMode)

        self.action_mini_mode_normal.triggered.connect(self.onActionMiniModeNormal)
        self.action_mini_mode_compact.triggered.connect(self.onActionMiniModeCompact)
        self.action_mini_mode_ultra.triggered.connect(self.onActionMiniModeUltra)
        self.action_select.triggered.connect(self.onActionSelect)
        self.action_select_viewport.triggered.connect(self.onActionSelectViewport)
        self.action_select_none.triggered.connect(self.onActionSelectNone)
        self.action_select_invert.triggered.connect(self.onActionSelectInvert)
        # action_zoom_out

        self.action_select_mode_replace.setCheckable(True)
        self.action_select_mode_and.setCheckable(True)
        self.action_select_mode_or.setCheckable(True)
        self.action_select_mode_xor.setCheckable(True)
        self.action_select_mode_subtract.setCheckable(True)

        # self.action_mini_mode.setCheckable(True)
        self.action_mini_mode_normal.setCheckable(True)
        self.action_mini_mode_normal.setChecked(True)
        self.action_mini_mode_compact.setCheckable(True)
        self.action_mini_mode_ultra.setCheckable(True)
        # self.action_mini_mode.setIcon(self.action_mini_mode_ultra.icon())
        # self.action_mini_mode.setText(self.action_mini_mode_ultra.text())

        self.action_move.setCheckable(True)
        self.action_pick.setCheckable(True)
        self.action_move.setChecked(True)
        self.action_select.setCheckable(True)
        self.action_xrange.setCheckable(True)
        self.action_yrange.setCheckable(True)
        self.action_lasso.setCheckable(True)
        self.action_select_rectangle.setCheckable(True)
        self.action_slice.setCheckable(True)
        # self.action_zoom_out.setCheckable(True)
        # self.action_group_main.

        # action = self.toolbar.addAction(icon
        self.syncToolbar()
        # self.action_select_mode_replace.setChecked(True)
        self.select_mode = "replace"
        self.setMode(self.action_move)
        self.toolbar.setIconSize(QtCore.QSize(16, 16))
        layout.addWidget(self.toolbar)

    def onActionDisplay(self, action):
        logger.debug("display = %r" % action.text())
        self.action_display_current = action
        self.plot()

    def onActionMiniMode(self):
        # targetAction = self.mini_mode_button.defaultAction()
        enabled_mini_mode = self.action_mini_mode_compact.isChecked() or self.action_mini_mode_ultra.isChecked()
        # enabled_mini_mode = self.action_mini_mode_normal.isChecked() or self.action_mini_mode_ultra.isChecked()
        ultra_mode = self.action_mini_mode_ultra.isChecked()

        logger.debug("mini screen: %r (ultra: %r)" % (enabled_mini_mode, ultra_mode))
        toolbuttons = self.toolbar.findChildren(QtGui.QToolButton)
        for toolbutton in toolbuttons:
            toolbutton.setToolButtonStyle(QtCore.Qt.ToolButtonIconOnly if enabled_mini_mode else QtCore.Qt.ToolButtonTextUnderIcon)

        if enabled_mini_mode:
            values = self.fig.subplotpars
            self.subplotpars_values = {"left": values.left, "right": values.right, "bottom": values.bottom, "top": values.top}
            self.bottomHeight = self.bottomFrame.height()

        self.bottomFrame.setVisible(not enabled_mini_mode)
        if 0:
            if enabled_mini_mode:
                self.resize(QtCore.QSize(self.width(), self.height() - self.bottomHeight))
            else:
                self.resize(QtCore.QSize(self.width(), self.height() + self.bottomHeight))
        self.fig.tight_layout()
        if enabled_mini_mode:
            if ultra_mode:
                self.fig.subplots_adjust(left=0, right=1, bottom=0, top=1.)
        else:
            self.fig.subplots_adjust(**self.subplotpars_values)
        self.canvas.draw()

    def on_toolbar_toggle(self, ignore=None):
        # self.action_toolbar_toggle.toggle()
        visible = self.action_toolbar_toggle.isChecked()
        logger.debug("toolbar visible? %r" % (visible, ))
        for widget in [self.toolbar, self.toolbar2, self.status_bar]:
            widget.setVisible(visible)

    def onActionMiniModeNormal(self, *args):
        self.onActionMiniMode()
        # self.mini_mode_button.setDefaultAction(self.action_miniscreen)
        # self.action_miniscreen.setChecked(True)
        # self.action_miniscreen_ultra.setChecked(False)
        # self.on
        # logger.debug("normal mini screen: %r" % self.action_miniscreen.isChecked())
        # self.action_mini_mode.setIcon(self.action_mini_mode_normal.icon())
        # self.action_mini_mode.setText(self.action_mini_mode_normal.text())
        # self.onActionMiniMode()
        # self.action_mini_mode.trigger()
        pass

    def onActionMiniModeCompact(self, *args):
        self.onActionMiniMode()

    def onActionMiniModeUltra(self, *args):
        self.onActionMiniMode()
        # self.mini_mode_button.setDefaultAction(self.action_miniscreen_ultra)
        # logger.debug("ultra mini screen: %r" % self.action_miniscreen_ultra.isChecked())
        # self.action_mini_mode.setIcon(self.action_mini_mode_ultra.icon())
        # self.action_mini_mode.setText(self.action_mini_mode_ultra.text())
        # self.action_mini_mode.trigger()
        # self.onActionMiniMode()
        # self.onActionMiniScreen()
        # self.action_miniscreen.setChecked(False)
        # self.action_miniscreen_ultra.setChecked(True)

    def setSelectMode(self, action):
        self.select_mode_button.setDefaultAction(action)
        if action == self.action_select_mode_replace:
            self._select_mode = "replace"
        if action == self.action_select_mode_and:
            self._select_mode = "and"
        if action == self.action_select_mode_or:
            self._select_mode = "or"
        if action == self.action_select_mode_xor:
            self._select_mode = "xor"
        if action == self.action_select_mode_subtract:
            self._select_mode = "subtract"

    @property
    def select_mode(self):
        return self._select_mode

    @select_mode.setter
    def select_mode(self, value):
        logger.debug("set to: %r", value)
        action = getattr(self, "action_select_mode_%s" % value)
        self.setSelectMode(action)

    def onActionSelectViewport(self):
        for layer in self.active_layers():
            if self.dimensions == 3:
                (xmin, xmax), (ymin, ymax), (zmin, zmax) = self.state.ranges_viewport
                args = (layer.x, xmin, layer.x, xmax, layer.y, ymin, layer.y, ymax, layer.z, zmin, layer.z, zmax)
                expression = "((%s) >= %f) & ((%s) <= %f) & ((%s) >= %f) & ((%s) <= %f)" % args
            if self.dimensions == 2:
                (xmin, xmax), (ymin, ymax) = self.state.ranges_viewport
                args = (layer.x, xmin, layer.x, xmax, layer.y, ymin, layer.y, ymax)
                expression = "((%s) >= %f) & ((%s) <= %f) & ((%s) >= %f) & ((%s) <= %f)" % args
            if self.dimensions == 1:
                (xmin, xmax) = self.state.ranges_viewport
                args = (layer.x, xmin, layer.x, xmax, layer.y, ymin, layer.y, ymax)
                expression = "((%s) >= %f) & ((%s) <= %f)" % args
            logger.debug("rectangle selection using expression: %r" % expression)
            layer.dataset.select(expression, mode=self.select_mode)
            # self.dataset.evaluate(select, layer.state.expressions[axes.xaxis_index], layer.state.expressions[axes.yaxis_index], **self.getVariableDict())
            mask = layer.dataset.mask
            self.queue_update()

    def onActionSelectNone(self):
        for layer in self.active_layers():
            layer.dataset.select(None)

    def onActionSelectInvert(self):
        for layer in self.active_layers():
            layer.dataset.select_inverse()
            self.queue_update()

    def onActionSelect(self):
        self.lastActionSelect.setChecked(True)
        self.setMode(self.lastActionSelect)
        self.syncToolbar()

    def syncToolbar(self):
        for plugin in self.plugins:
            plugin.syncToolbar()
        for action in [self.action_select]:  # , self.action_zoom]:
            logger.debug("sync action: %r" % action.text())
            subactions = action.menu().actions()
            subaction_selected = [subaction for subaction in subactions if subaction.isChecked()]
            # if len(subaction_selected) > 0:
            #   action.setText(subaction_selected[0].text())
            #   action.setIcon(subaction_selected[0].icon())
            logger.debug(" subaction_selected: %r" % subaction_selected)
            logger.debug(" action was selected?: %r" % action.isChecked())
            action.setChecked(len(subaction_selected) > 0)
            logger.debug(" action  is selected?: %r" % action.isChecked())
        logger.debug("last select action: %r" % self.lastActionSelect.text())
        # logger.debug("last zoom action: %r" % self.lastActionZoom.text())
        self.action_select.setText(self.lastActionSelect.text())
        self.action_select.setIcon(self.lastActionSelect.icon())
        # self.action_zoom.setText(self.lastActionZoom.text())
        # self.action_zoom.setIcon(self.lastActionZoom.icon())
        # self.action_select.update()

    def check_aspect(self, axis_follow):
        if self.state.aspect is not None:
            if self.state.ranges_viewport is None:
                return
            if any([k is None for k in self.state.ranges_viewport]):
                return
            width = self.state.ranges_viewport[axis_follow][1] - self.state.ranges_viewport[axis_follow][0]
            centers = [(self.state.ranges_viewport[i][1] + self.state.ranges_viewport[i][0]) / 2. for i in range(self.dimensions)]
            for i in range(self.dimensions):
                if i != axis_follow:
                    self.state.ranges_viewport[i] = [None, None]
                    self.state.ranges_viewport[i][0] = centers[i] - width / 2
                    self.state.ranges_viewport[i][1] = centers[i] + width / 2
            return

            otheraxes = list(range(self.dimensions))
            allaxes = list(range(self.dimensions))
            otheraxes.remove(axis_follow)
            # ranges = [self.state.ranges_viewport[i] if self.state.ranges_viewport[i] is not None else self.ranges[i] for i in otheraxes]
            ranges = [self.state.ranges_viewport[i] for i in otheraxes]

            logger.debug("aspect 1")
            if None in ranges:
                return
            width = self.state.ranges_viewport[axis_follow][1] - self.state.ranges_viewport[axis_follow][0]
            # width = ranges[axis_follow][1] - ranges[axis_follow][0]
            center = (self.state.ranges_viewport[axis_follow][1] + self.state.ranges_viewport[axis_follow][0]) / 2.
            logger.debug("aspect 2")

            # widths = [ranges[i][1] - ranges[i][0] for i in range(self.dimensions-1)]
            centers = [(self.state.ranges_viewport[i][1] + self.state.ranges_viewport[i][0]) / 2. for i in range(self.dimensions)]
            logger.debug("aspect 3")

            # xmin, xmax = self.ranges[0]
            # ymin, ymax = self.ranges[1]
            for i in range(self.dimensions - 1):
                axis_index = otheraxes[i]
                # if self.state.ranges_viewport[i] is None:
                #   self.state.ranges_viewport[i] = self.ranges[i]
                self.state.ranges_viewport[axis_index] = [None, None]
                self.state.ranges_viewport[axis_index][0] = centers[axis_index] - width / 2
                self.state.ranges_viewport[axis_index][1] = centers[axis_index] + width / 2
                logger.debug("aspect i=%d,%d", i, axis_index)
            for layer in self.layers:
                for i in range(self.dimensions - 1):
                    axis_index = otheraxes[i]
                    # layer.state.ranges_grid[axis_index] = list(self.state.ranges_viewport[axis_index])
                    layer.set_range(list(self.state.ranges_viewport[axis_index]), axis_index)
                layer.state.ranges_grid[axis_follow] = list(self.state.ranges_viewport[axis_follow])

class HistogramPlotDialog(PlotDialog):
    type_name = "histogram"
    # names = "histogram,1d"

    def __init__(self, parent, dataset, app, **kwargs):
        super(HistogramPlotDialog, self).__init__(parent, dataset, 1, ["x"], app, **kwargs)

    def beforeCanvas(self, layout):
        self.addToolbar(layout, yselect=False, lasso=False)

    def _afterCanvas(self, layout):
        self.addToolbar2(layout, contrast=False, gamma=False)
        super(HistogramPlotDialog, self).afterCanvas(layout)

    def add_histogram(self, x, counts, selection):
        self.histograms.append((x, counts))

    def plot(self):
        t0 = time.time()
        self.axes.cla()
        self.axes.autoscale(False)
        # if self.expression_error:
        #   return
        # P.hist(x, 50, normed=1, histtype='stepfilled')
        # values =
        if len(self.layers) == 0:
            return
        first_layer = self.layers[0]
        if self.state.range_level_show is None:
            self.state.range_level_show = first_layer.range_level
        if self.state.range_level_show is None:
            logger.error("cannot plot when range_level_show is None")
            return

        # for i in range(self.dimensions):
        #   if self.state.ranges_viewport[i] is None:
        #       self.state.ranges_viewport[i] = copy.copy(first_layer.state.ranges_grid[i])

        for layer in self.layers:
            layer.plot([self.axes], self.add_histogram)
            # layer.prepare_histograms()

        # self.axes.set_xlabel(first_layer.state.expressions[0])
        xmin_show, xmax_show = self.state.ranges_viewport[0]
        self.axes.set_xlim(xmin_show, xmax_show)
        # if self.state.range_level_show is None:
        # self.state.range_level_show = first_layer.range_level
        ymin_show, ymax_show = self.state.range_level_show
        # if not first_layer.state.weight_expression:
        # self.axes.set_ylabel("counts")
        # else:
        # self.axes.set_ylabel(first_layer.state.weight_expression)
        self.axes.set_xlabel(self.get_label(0))
        self.axes.set_ylabel(self.get_label(1))
        self.axes.set_ylim(ymin_show, ymax_show)
        if not self.action_mini_mode_ultra.isChecked():
            self.fig.tight_layout(pad=0.0)
        self.canvas.draw()
        self.update()
        self.message("plotting %.2fs" % (time.time() - t0), index=100)
        self.signal_plot_finished.emit(self, self.fig)
        if self._plot_event:
            logger.debug("plotting done")
            self._plot_event.set()
        return

        Nvector = self.state.grid_size
        width = self.state.ranges_viewport[0][1] - self.state.ranges_viewport[0][0]
        x = np.arange(0, Nvector) / float(Nvector) * width + self.state.ranges_viewport[0][0]  # + width/(Nvector/2.)
        xmin, xmax = self.state.ranges_viewport[0]
        xmin, xmax = self.state.ranges_viewport[0]
        if self.state.ranges_viewport[0] is None:
            self.state.ranges_viewport[0] = [xmin, xmax]

        self.delta = (xmax - xmin) / self.state.grid_size
        self.centers = (np.arange(self.state.grid_size) + 0.5) * self.delta + xmin

        logger.debug("expr for amplitude: %r" % self.amplitude_expression)
        grid_map = self.create_grid_map(self.state.grid_size, False)
        amplitude = self.eval_amplitude(self.amplitude_expression, locals=grid_map)
        use_selection = self.dataset.mask is not None
        if use_selection:
            grid_map_selection = self.create_grid_map(self.state.grid_size, True)
            amplitude_selection = self.eval_amplitude(self.amplitude_expression, locals=grid_map_selection)

        if use_selection:
            self.axes.bar(self.centers, amplitude, width=self.delta, align='center')
            self.axes.bar(self.centers, amplitude_selection, width=self.delta, align='center', color="red", alpha=0.8)
        else:
            self.axes.bar(self.centers, amplitude, width=self.delta, align='center')

        if self.range_level is None:
            if self.state.weight_expression:
                self.range_level = np.nanmin(amplitude) * 1.1, np.nanmax(amplitude) * 1.1
            else:
                self.range_level = 0, np.nanmax(amplitude) * 1.1
        if self.action_mini_mode_compact:
            self.fig.subplots_adjust(left=0, right=1, bottom=0, top=1.)

class ScatterPlotDialog(PlotDialog):
    type_name = "density2d"
    # names = "heatmap,density2d,2d"

    def __init__(self, parent, dataset, app, **options):
        super(ScatterPlotDialog, self).__init__(parent, dataset, 2, "x y".split(), app, **options)

    def _afterCanvas(self, layout):
        self.addToolbar2(layout)
        super(ScatterPlotDialog, self).afterCanvas(layout)

    def add_image_layer(self, rgba, intensity):
        self.image_layers.append(rgba)

    def post_draw(self, fig, axes):
        pass

    def plot(self):
        self.fig.clf()
        self.add_axes()
        if 0:  # hasattr(self, "colorbar") and self.colorbar:
            # self.fig.delaxes(self.colorbar.ax)
            self.fig.delaxes(self.fig.axes[1])
            self.colorbar = None
        self.image_layers = []
        self.axes.rgb_images = self.image_layers
        self.axes.cla()
        if len(self.layers) == 0:
            self.canvas.draw()
            self.update()
            return
        first_layer = self.layers[0]

        N = self.state.grid_size
        background = np.ones((N, N, 4), dtype=np.float64)
        background_color = self.background_color
        if background_color == "auto":
            if self.blend_mode in "screen lighten".split():
                background_color = "black"
            else:
                background_color = "white"
        background[:, :, 0:3] = matplotlib.colors.colorConverter.to_rgb(background_color)
        background[:, :, 3] = 1.

        ranges = []
        # this happens when for instance the plotting isn't done, and you change the display %
        if first_layer.state.ranges_grid is None:
            return
        if None in first_layer.state.ranges_grid:
            return
        for minimum, maximum in first_layer.state.ranges_grid:
            ranges.append(minimum)
            ranges.append(maximum)

        background = np.transpose(background, (1, 0, 2))
        placeholder = self.axes.imshow(background, extent=ranges, origin="lower")
        self.add_image_layer(background, None)

        for i in range(self.dimensions):
            if self.state.ranges_viewport[i] is None:
                self.state.ranges_viewport[i] = copy.copy(first_layer.state.ranges_grid[i])
        # extent =
        # ranges = np.nanmin(datax), np.nanmax(datax), np.nanmin(datay), np.nanmax(datay)

        xmin, xmax = self.state.ranges_viewport[0]
        ymin, ymax = self.state.ranges_viewport[1]
        width = xmax - xmin
        height = ymax - ymin
        extent = [xmin - width, xmax + width, ymin - height, ymax + height]
        Z1 = np.array(([0, 1] * 8 + [1, 0] * 8) * 8)
        Z1.shape = 16, 16  # chessboard
        # im1 = self.axes.imshow(Z1, cmap="gray", interpolation='nearest', extent=extent, vmin=-4, vmax=1.)

        for layer in self.layers:
            layer.plot([self.axes], self.add_image_layer)

        rgba = vaex.ui.imageblending.blend(self.image_layers, self.blend_mode)
        rgba[..., 3] = rgba[..., 3] * 0 + 1
        for c in range(4):
            # rgba_dest[:,:,c] = np.clip((rgba_dest[:,:,c] ** 3.5)*2.6, 0., 1.)
            rgba[:, :, c] = np.clip((rgba[:, :, c] ** self.layer_gamma) * self.layer_brightness, 0., 1.)
        rgba = np.transpose(rgba, (1, 0, 2))
        placeholder.set_data((rgba * 255).astype(np.uint8))

        if self.state.aspect is None:
            self.axes.set_aspect('auto')
        else:
            self.axes.set_aspect(self.state.aspect)
            # if self.dataset.selected_row_index is not None:
            # self.axes.autoscale(False)
        # index = self.dataset.selected_row_index

        if 1:

            self.axes.set_xlabel(self.get_label(0))
            self.axes.set_ylabel(self.get_label(1))
        self.axes.set_xlim(*self.state.ranges_viewport[0])
        self.axes.set_ylim(*self.state.ranges_viewport[1])
        layer = first_layer

        class Dummy(object):
            pass

        class ColorbarWrapper(matplotlib.cm.ScalarMappable):
            def __init__(self, layer):
                self.layer = layer
                self.cmap = matplotlib.cm.cmap_d[self.layer.state.colormap]
                self.norm = matplotlib.colors.Normalize(layer.level_ranges[0], layer.level_ranges[1])
                self.norm = matplotlib.colors.Normalize(max(layer.level_ranges), min(layer.level_ranges))
                matplotlib.cm.ScalarMappable.__init__(self, self.norm, self.cmap)

            def autoscale_None(self):
                pass

        if first_layer.state.colorbar:
            wrapper = ColorbarWrapper(first_layer)
            self.colorbar = self.fig.colorbar(wrapper, use_gridspec=True)
            self.colorbar.ax.set_ylabel(first_layer.amplitude_label())
            ymin, ymax = self.colorbar.ax.get_ylim()
            if (layer.level_ranges[0] > layer.level_ranges[1]) != (ymin > ymax):
                self.colorbar.ax.invert_yaxis()

        # self.fig.texts = []
        # if first_layer.flip_x:
        # self.axes.invert_xaxis()
        # if first_layer.flip_y:
        # self.axes.invert_yaxis()
        if 0:
            title_text = self.title_expression.format(**self.getVariableDict())
            if hasattr(self, "title"):
                self.title.set_text(title_text)
            else:
                self.title = self.fig.suptitle(title_text)
            self.canvas.draw()
            # self.fig.tight_layout(pad=0.0)#1.008) #pad=pad, h_pad=h_pad, w_pad=w_pad, rect=rect)
            # self.fig.tight_layout(pad=0.01)#1.008) #pad=pad, h_pad=h_pad, w_pad=w_pad, rect=rect)
        if self.action_mini_mode_ultra.isChecked():
            logger.debug("ultra compact mode")
            self.fig.subplots_adjust(left=0, right=1, bottom=0, top=1.)
        # print(self.fig.axes)
        # for axes in self.fig.axes:
        # print("      %s" % axes)
        # print self.fig._axstack()

        class ScaledLocator(matplotlib.ticker.MaxNLocator):
            """
            Locates regular intervals along an axis scaled by *dx* and shifted by
            *x0*. For example, this would locate minutes on an axis plotted in seconds
            if dx=60.  This differs from MultipleLocator in that an approriate interval
            of dx units will be chosen similar to the default MaxNLocator.
            """

            def __init__(self, dx=1.0, x0=0.0):
                self.dx = dx
                self.x0 = x0
                matplotlib.ticker.MaxNLocator.__init__(self, nbins=9, steps=[1, 2, 5, 10])

            def rescale(self, x):
                return x / self.dx + self.x0

            def inv_rescale(self, x):
                return (x - self.x0) * self.dx

            def __call__(self):
                vmin, vmax = self.axis.get_view_interval()
                vmin, vmax = self.rescale(vmin), self.rescale(vmax)
                vmin, vmax = matplotlib.transforms.nonsingular(vmin, vmax, expander=0.05)
                locs = self.bin_boundaries(vmin, vmax)
                locs = self.inv_rescale(locs)
                prune = self._prune
                if prune == 'lower':
                    locs = locs[1:]
                elif prune == 'upper':
                    locs = locs[:-1]
                elif prune == 'both':
                    locs = locs[1:-1]
                return self.raise_if_exceeds(locs)

        class ScaledFormatter(matplotlib.ticker.OldScalarFormatter):
            """Formats tick labels scaled by *dx* and shifted by *x0*."""

            def __init__(self, dx=1.0, x0=0.0, **kwargs):
                self.dx, self.x0 = dx, x0

            def rescale(self, x):
                return x / self.dx + self.x0

            def __call__(self, x, pos=None):
                xmin, xmax = self.axis.get_view_interval()
                xmin, xmax = self.rescale(xmin), self.rescale(xmax)
                d = abs(xmax - xmin)
                x = self.rescale(x)
                s = self.pprint_val(x, d)
                return s
        for dim in range(2):
            if first_layer.state.output_units[dim]:
                unit = first_layer.dataset.unit(first_layer.state.expressions[dim])
                try:
                    scale = astropy.units.Unit(first_layer.state.output_units[dim]).to(unit)
                    axis = self.axes.xaxis if dim == 0 else self.axes.yaxis
                    axis.set_major_locator(ScaledLocator(dx=scale))
                    axis.set_major_formatter(ScaledFormatter(dx=scale))
                except:
                    pass

        titles = [layer.state.title for layer in self.layers if layer.state.title]
        if titles:
            self.title = self.axes.set_title(",".join(titles))
        self.post_draw(self.fig, self.axes)
        self.canvas.draw()
        self.fig.tight_layout()  # 1.008) #pad=pad, h_pad=h_pad, w_pad=w_pad, rect=rect)
        self.canvas.draw()
        self.update()
        if 0:
            if self.first_time:
                self.first_time = False
                if "filename" in self.options:
                    self.filename_figure_last = self.options["filename"]
                    self.fig.savefig(self.filename_figure_last)

        # this will readd any widgets
        self.setMode(self.lastAction)
        self.signal_plot_finished.emit(self, self.fig)
        logger.debug("plot finished")
        if self._plot_event:
            self._plot_event.set()
        # traceback.print_stack()
        return
        if 1:
            ranges = []
            logger.debug("self.ranges == %r" % (self.ranges, ))
            for minimum, maximum in self.ranges:
                ranges.append(minimum)
                ranges.append(maximum)

            # amplitude = self.grids.grids["counts"].get_data(self.gridsize)

            logger.debug("expr for amplitude: %r" % self.amplitude_expression)
            grid_map = self.create_grid_map(self.state.grid_size, False)
            try:
                amplitude = self.eval_amplitude(self.amplitude_expression, locals=grid_map)
            except Exception as e:
                self.error_in_field(self.amplitude_box, "amplitude", e)
                return
            use_selection = self.dataset.mask is not None
            if use_selection:
                grid_map_selection = self.create_grid_map(self.state.grid_size, True)
                amplitude_selection = self.eval_amplitude(self.amplitude_expression, locals=grid_map_selection)

        if self.action_display_current == self.action_display_mode_both:
            self.axes.imshow(self.contrast(amplitude), origin="lower", extent=ranges, alpha=0.4 if use_selection else 1.0, cmap=self.colormap)
            if use_selection:
                self.axes.imshow(self.contrast(amplitude_selection), origin="lower", extent=ranges, alpha=1, cmap=self.colormap)
        if self.action_display_current == self.action_display_mode_full:
            self.axes.imshow(self.contrast(amplitude), origin="lower", extent=ranges, cmap=self.colormap)
        if self.action_display_current == self.action_display_mode_selection:
            if self.counts_mask is not None:
                self.axes.imshow(self.contrast(amplitude_mask), origin="lower", extent=ranges, alpha=1, cmap=self.colormap)
        if 1:
            # locals = {key:None if grid is None else vaex.vaexfast.resize(grid, 64) for key, grid in locals}
            locals = {}
            for name in list(self.grids.grids.keys()):
                grid = self.grids.grids[name]
                if name == "counts" or (grid.weight_expression is not None and len(grid.weight_expression) > 0):
                    if grid.max_size >= self.state.vector_grid_size:
                        locals[name] = grid.get_data(self.state.vector_grid_size, use_selection)
                else:
                    locals[name] = None

            if 1:
                grid_map_vector = self.create_grid_map(self.state.vector_grid_size, use_selection)
                if grid_map_vector["weightx"] is not None and grid_map_vector["weighty"] is not None:
                    mask = grid_map_vector["counts"] > (self.min_level_vector2d * grid_map_vector["counts"].max())
                    x = grid_map_vector["x"]
                    y = grid_map_vector["y"]
                    x2d, y2d = np.meshgrid(x, y)
                    vx = self.eval_amplitude("weightx/counts", locals=grid_map_vector)
                    vy = self.eval_amplitude("weighty/counts", locals=grid_map_vector)
                    meanvx = 0 if self.vectors_subtract_mean is False else vx[mask].mean()
                    meanvy = 0 if self.vectors_subtract_mean is False else vy[mask].mean()
                    vx -= meanvx
                    vy -= meanvy
                    if grid_map_vector["weightz"] is not None and self.vectors_color_code_3rd:
                        colors = self.eval_amplitude("weightz/counts", locals=grid_map_vector)
                        self.axes.quiver(x2d[mask], y2d[mask], vx[mask], vy[mask], colors[mask], cmap=self.colormap_vector)  # , scale=1)
                    else:
                        self.axes.quiver(x2d[mask], y2d[mask], vx[mask], vy[mask], color="black")
                        colors = None
                # self.axes.quiver(x, y, U, V)
        if self.action_display_current == self.action_display_mode_both_contour:
            # self.axes.imshow(amplitude, origin="lower", extent=ranges, alpha=1 if self.counts_mask is None else 0.4, cmap=cm_plusmin)
            # self.axes.contour(amplitude, origin="lower", extent=ranges, levels=levels, linewidths=2, colors="red")
            self.axes.imshow(amplitude, origin="lower", extent=ranges, cmap=self.colormap)
            if self.counts_mask is not None:
                values = amplitude_mask[~np.isinf(amplitude_mask)]
                levels = np.linspace(values.min(), values.max(), 5)
                # self.axes.imshow(amplitude_mask, origin="lower", extent=ranges, alpha=1, cmap=cm_plusmin)
                self.axes.contour(amplitude_mask, origin="lower", extent=ranges, levels=levels, linewidths=2, colors="red")

        for callback in self.plugin_grids_draw:
            callback(self.axes, grid_map, grid_map_vector)

        if self.state.aspect is None:
            self.axes.set_aspect('auto')
        else:
            self.axes.set_aspect(self.state.aspect)
            # if self.dataset.selected_row_index is not None:
            # self.axes.autoscale(False)
        if 0:
            index = self.dataset.selected_row_index
            if index is not None and self.selected_point is None:
                logger.debug("point selected but after computation")
                # TODO: optimize

                def find_selected_point(info, blockx, blocky):
                    if index >= info.i1 and index < info.i2:  # selected point is in this block
                        self.selected_point = blockx[index - info.i1], blocky[index - info.i1]
                self.dataset.evaluate(find_selected_point, *self.expressions, **self.getVariableDict())

            if self.selected_point:
                # x, y = self.getdatax()[self.dataset.selected_row_index],  self.getdatay()[self.dataset.selected_row_index]
                x, y = self.selected_point
                self.axes.scatter([x], [y], color='red')  # , scalex=False, scaley=False)
            # if dataxsel is not None:
            #   self.axes.scatter(dataxsel, dataysel)
        self.axes.set_xlabel(self.expressions[0])
        self.axes.set_ylabel(self.expressions[1])
        self.axes.set_xlim(*self.state.ranges_viewport[0])
        self.axes.set_ylim(*self.state.ranges_viewport[1])
        # self.fig.texts = []
        title_text = self.title_expression.format(**self.getVariableDict())
        if hasattr(self, "title"):
            self.title.set_text(title_text)
        else:
            self.title = self.fig.suptitle(title_text)
        if not self.action_mini_mode_ultra.isChecked():
            self.fig.tight_layout(pad=0.0)  # 1.008) #pad=pad, h_pad=h_pad, w_pad=w_pad, rect=rect)
        # self.fig.tight_layout(pad=0.01)#1.008) #pad=pad, h_pad=h_pad, w_pad=w_pad, rect=rect)
        self.fig.tight_layout()  # 1.008) #pad=pad, h_pad=h_pad, w_pad=w_pad, rect=rect)
        self.canvas.draw()
        self.update()
        if self.first_time:
            self.first_time = False
            if "filename" in self.options:
                self.filename_figure_last = self.options["filename"]
                self.fig.savefig(self.filename_figure_last)

class ScatterPlotMatrixDialog(PlotDialog):
    def __init__(self, parent, dataset, expressions):
        super(ScatterPlotMatrixDialog, self).__init__(parent, dataset, list(expressions), "X Y Z W V U T S R Q P".split()[:len(expressions)])

    def getAxesList(self):
        return reduce(lambda x, y: x + y, self.axes_grid, [])

    def addAxes(self):
        self.axes_grid = [[None, ] * self.dimensions for _ in range(self.dimensions)]
        index = 0
        for i in range(self.dimensions)[::1]:
            for j in range(self.dimensions)[::1]:
                index = ((self.dimensions - 1) - j) * self.dimensions + i + 1
                axes = self.axes_grid[i][j] = self.fig.add_subplot(self.dimensions, self.dimensions, index)
                # sharey=self.axes_grid[0][j] if j > 0 else None,
                # sharex=self.axes_grid[i][0] if i > 0 else None)
                # store the axis index in matplotlib object
                axes.xaxis_index = i
                axes.yaxis_index = j
                if i > 0:
                    axes.yaxis.set_visible(False)
                    # for label in axes.get_yticklabels():
                    # label.set_visible(False)
                    # axes.yaxis.offsetText.set_visible(False)
                if j > 0:
                    axes.xaxis.set_visible(False)
                    # for label in axes.get_xticklabels():
                    # label.set_visible(False)
                    # axes.xaxis.offsetText.set_visible(False)
                if int(matplotlib.__version__.split('.')[0]) < 3:
                    self.axes_grid[i][j].hold(True)
                index += 1
        self.fig.subplots_adjust(hspace=0, wspace=0)

    def calculate_visuals(self, info, *blocks):
        data_blocks = blocks[:self.dimensions]
        if len(blocks) > self.dimensions:
            weights_block = blocks[self.dimensions]
        else:
            weights_block = None
        elapsed = time.time() - info.time_start
        self.message("computation %.2f%% (%f seconds)" % (info.percentage, elapsed), index=9)
        QtCore.QCoreApplication.instance().processEvents()
        self.expression_error = False

        N = self.state.grid_size
        mask = self.dataset.mask
        if info.first:
            self.counts = np.zeros((N,) * self.dimensions, dtype=np.float64)
            self.counts_weights = self.counts
            if weights_block is not None:
                self.counts_weights = np.zeros((N,) * self.dimensions, dtype=np.float64)

            self.selected_point = None
            if mask is not None:
                self.counts_mask = np.zeros((N,) * self.dimensions, dtype=np.float64)  # mab.utils.numpy.mmapzeros((128), dtype=np.float64)
                self.counts_weights_mask = self.counts_mask
                if weights_block is not None:
                    self.counts_weights_mask = np.zeros((N,) * self.dimensions, dtype=np.float64)
            else:
                self.counts_mask = None
                self.counts_weights_mask = None

        if info.error:
            self.expression_error = True
            self.message(info.error_text)
            return

        xmin, xmax = self.ranges[0]
        ymin, ymax = self.ranges[1]
        for i in range(self.dimensions):
            if self.state.ranges_viewport[i] is None:
                self.state.ranges_viewport[i] = self.ranges[i]

        index = self.dataset.selected_row_index
        if index is not None:
            if index >= info.i1 and index < info.i2:  # selected point is in this block
                self.selected_point = blockx[index - info.i1], blocky[index - info.i1]

        t0 = time.time()
        # histo2d(blockx, blocky, self.counts, *self.ranges)
        ranges = []
        for minimum, maximum in self.ranges:
            ranges.append(minimum)
            if minimum == maximum:
                maximum += 1
            ranges.append(maximum)
        try:
            args = data_blocks, self.counts, ranges
            if self.dimensions == 2:
                vaex.histogram.hist3d(data_blocks[0], data_blocks[1], self.counts, *ranges)
            if self.dimensions == 3:
                vaex.histogram.hist3d(data_blocks[0], data_blocks[1], data_blocks[2], self.counts, *ranges)
            if weights_block is not None:
                args = data_blocks, weights_block, self.counts, ranges
                vaex.histogram.hist2d_weights(blockx, blocky, self.counts_weights, weights_block, *ranges)
        except:
            raise

        if mask is not None:
            subsets = [block[mask[info.i1:info.i2]] for block in data_blocks]
            if self.dimensions == 2:
                vaex.histogram.hist2d(subsets[0], subsets[1], self.counts_weights_mask, *ranges)
            if self.dimensions == 3:
                vaex.histogram.hist3d(subsets[0], subsets[1], subsets[2], self.counts_weights_mask, *ranges)
            if weights_block is not None:
                subset_weights = weights_block[mask[info.i1:info.i2]]
                if self.dimensions == 2:
                    vaex.histogram.hist2d_weights(subsets[0], subsets[1], self.counts_weights_mask, subset_weights, *ranges)
                if self.dimensions == 3:
                    vaex.histogram.hist3d_weights(subsets[0], subsets[1], subsets[2], self.counts_weights_mask, subset_weights, *ranges)
        if info.last:
            elapsed = time.time() - info.time_start
            self.message("computation (%f seconds)" % (elapsed), index=9)

    def plot(self):
        t0 = time.time()
        # self.axes.cla()
        # extent =
        # ranges = np.nanmin(datax), np.nanmax(datax), np.nanmin(datay), np.nanmax(datay)
        ranges = []
        for minimum, maximum in self.ranges:
            ranges.append(minimum)
            ranges.append(maximum)

        amplitude = self.counts
        logger.debug("expr for amplitude: %r" % self.amplitude_expression)
        if self.amplitude_expression is not None:
            locals = {"counts": self.counts_weights, "counts1": self.counts}
            globals = np.__dict__
            amplitude = eval(self.amplitude_expression, globals, locals)
        # if self.ranges_level[0] is None:
        # self.ranges_level[0] = 0, amplitude.max() * 1.1

        def multisum(a, axes):
            correction = 0
            for axis in axes:
                a = np.sum(a, axis=axis - correction)
                correction += 1
            return a
        for i in range(self.dimensions):
            for j in range(self.dimensions):
                axes = self.axes_grid[i][j]
                ranges = self.ranges[i] + self.ranges[j]
                axes.clear()
                allaxes = list(range(self.dimensions))
                if 0:  # i > 0:
                    for label in axes.get_yticklabels():
                        label.set_visible(False)
                    axes.yaxis.offsetText.set_visible(False)
                if 0:  # j > 0:
                    for label in axes.get_xticklabels():
                        label.set_visible(False)
                    axes.xaxis.offsetText.set_visible(False)
                if i != j:
                    allaxes.remove(i)
                    allaxes.remove(j)
                    counts_mask = None
                    counts = multisum(self.counts, allaxes)
                    if self.counts_mask is not None:
                        counts_mask = multisum(self.counts_mask, allaxes)
                    if i > j:
                        counts = counts.T
                    axes.imshow(np.log10(counts), origin="lower", extent=ranges, alpha=1 if counts_mask is None else 0.4)
                    if counts_mask is not None:
                        if i > j:
                            counts_mask = counts_mask.T
                        axes.imshow(np.log10(counts_mask), origin="lower", extent=ranges)
                    axes.set_aspect('auto')
                    if self.dataset.selected_row_index is not None:
                        # self.axes.autoscale(False)
                        x, y = self.getdatax()[self.dataset.selected_row_index], self.getdatay()[self.dataset.selected_row_index]
                        axes.scatter([x], [y], color='red')  # , scalex=False, scaley=False)

                    axes.set_xlim(self.state.ranges_viewport[i][0], self.state.ranges_viewport[i][1])
                    axes.set_ylim(self.state.ranges_viewport[j][0], self.state.ranges_viewport[j][1])
                else:
                    allaxes.remove(j)
                    counts = multisum(self.counts, allaxes)
                    N = len(counts)
                    xmin, xmax = self.ranges[i]
                    delta = (xmax - xmin) / N
                    centers = np.arange(N) * delta + xmin

                    # axes.autoscale(False)
                    # P.hist(x, 50, normed=1, histtype='stepfilled')
                    # values =
                    if 1:  # if self.counts_mask is None:
                        axes.bar(centers, counts, width=delta, align='center')
                    else:
                        self.axes.bar(self.centers, self.counts, width=self.delta, align='center', alpha=0.5)
                        self.axes.bar(self.centers, self.counts_mask, width=self.delta, align='center', color="red")
                    axes.set_xlim(self.state.ranges_viewport[i][0], self.state.ranges_viewport[i][1])
                    axes.set_ylim(0, np.max(counts) * 1.1)

        if 0:

            self.axes.imshow(amplitude.T, origin="lower", extent=ranges, alpha=1 if self.counts_mask is None else 0.4, cmap=cm_plusmin)
            if 1:
                if self.counts_mask is not None:
                    if self.amplitude_expression is not None:
                        # locals = {"counts":self.counts_mask}
                        locals = {"counts": self.counts_weights_mask, "counts1": self.counts_mask}
                        globals = np.__dict__
                        amplitude_mask = eval(self.amplitude_expression, globals, locals)
                    self.axes.imshow(amplitude_mask.T, origin="lower", extent=ranges, alpha=1, cmap=cm_plusmin)
                # self.axes.imshow((I), origin="lower", extent=ranges)
            self.axes.set_aspect('auto')
            # if self.dataset.selected_row_index is not None:
            # self.axes.autoscale(False)
            index = self.dataset.selected_row_index
            if index is not None and self.selected_point is None:
                logger.debug("point selected but after computation")
                # TODO: optimize

                def find_selected_point(info, blockx, blocky):
                    if index >= info.i1 and index < info.i2:  # selected point is in this block
                        self.selected_point = blockx[index - info.i1], blocky[index - info.i1]
                self.dataset.evaluate(find_selected_point, *self.expressions, **self.getVariableDict())

            if self.selected_point:
                # x, y = self.getdatax()[self.dataset.selected_row_index],  self.getdatay()[self.dataset.selected_row_index]
                x, y = self.selected_point
                self.axes.scatter([x], [y], color='red')  # , scalex=False, scaley=False)
            # if dataxsel is not None:
            # self.axes.scatter(dataxsel, dataysel)
            self.axes.set_xlabel(self.expressions[0])
            self.axes.set_ylabel(self.expressions[0])
            self.axes.set_xlim(*self.state.ranges_viewport[0])
            self.axes.set_ylim(*self.state.ranges_viewport[1])
        self.canvas.draw()
        self.message("ploting %f" % (time.time() - t0), index=5)

time_previous = time.time()
time_start = time.time()

def timelog(msg, reset=False):
    global time_previous, time_start
    now = time.time()
    if reset:
        time_start = now
    T = now - time_start
    deltaT = now - time_previous
    logger.info("*** TIMELOG: %s (T=%f deltaT=%f)" % (msg, T, deltaT))
    time_previous = now

class VolumeRenderingPlotDialog(PlotDialog):
    type_name = "volumerendering"
    # names = "volumerendering,3d"

    def __init__(self, parent, dataset, app, **options):
        super(VolumeRenderingPlotDialog, self).__init__(parent, dataset, 3, "x y x".split(), app, **options)
        # [xname, yname, zname]

    def closeEvent(self, event):
        self.widget_volume.orbit_stop()
        super(VolumeRenderingPlotDialog, self).closeEvent(event)

    def afterCanvas(self, layout):

        self.widget_volume = vaex.ui.volumerendering.VolumeRenderWidget(self)
        self.layout_plot_region.insertWidget(0, self.widget_volume, 1)

        # self.addToolbar2(layout)
        super(VolumeRenderingPlotDialog, self).afterCanvas(layout)

        self.menu_view.addSeparator()
        self.action_group_quality_3d = QtGui.QActionGroup(self)
        self.actions_quality_3d = []
        for index, (name, resolution, iterations) in enumerate([("Fast", 256, 200), ("Medium", 256 + 128, 300), ("Best", 512, 500)]):
            if "vr_quality" in self.options:
                vr_index = int(eval(self.options.get("vr_quality")))
                if index == vr_index:
                    # self.actions_quality_3d[index].trigger()
                    self.widget_volume.ray_iterations = iterations
                    self.widget_volume.texture_size = resolution

            action_quality_3d = QtGui.QAction(QtGui.QIcon(iconfile('picture_empty')), name + ' volume rendering', self)

            def do(ignore=None, resolution=resolution, iterations=iterations):
                self.widget_volume.set_iterations(iterations)
                self.widget_volume.setResolution(resolution)
                self.widget_volume.update()
            action_quality_3d.setCheckable(True)
            if resolution == self.widget_volume.texture_size and iterations == self.widget_volume.ray_iterations:
                action_quality_3d.setChecked(True)
            action_quality_3d.triggered.connect(do)
            action_quality_3d.setShortcut("Ctrl+Shift+Meta+%d" % (index + 1))
            self.menu_view.addAction(action_quality_3d)
            self.action_group_quality_3d.addAction(action_quality_3d)
            self.actions_quality_3d.append(action_quality_3d)

    def getAxesList(self):
        # return reduce(lambda x,y: x + y, self.axes_grid, [])
        return [self.axis_top, self.axis_bottom]

    def add_pages(self, toolbox):
        self.frame_options_volume_rendering = QtGui.QFrame(self)

        toolbox.addItem(self.frame_options_volume_rendering, "Volume rendering")
        toolbox.setCurrentIndex(3)
        self.fill_page_volume_rendering(self.frame_options_volume_rendering)

    def add_axes(self):
        self.axes_grid = [[None, ] * self.dimensions for _ in range(self.dimensions)]
        self.axis_top = self.fig.add_subplot(2, 1, 1)
        self.axis_bottom = self.fig.add_subplot(2, 1, 2)
        self.axis_top.xaxis_index = 0
        self.axis_top.yaxis_index = 1
        self.axis_bottom.xaxis_index = 0
        self.axis_bottom.yaxis_index = 2
        # self.fig.subplots_adjust(hspace=0, wspace=0)

    def add_image_layer(self, rgba, intensity):
        self.image_layers.append(intensity)

    def plot(self):
        self.image_layers = []
        # self.image_layers = []

        axes_list = self.getAxesList()
        for axes in axes_list:
            axes.cla()
        if len(self.layers) == 0:
            return
        first_layer = self.layers[0]

        for i in range(self.dimensions):
            if self.state.ranges_viewport[i] is None:
                self.state.ranges_viewport[i] = copy.copy(first_layer.state.ranges_grid[i])
        # extent =
        for axes in axes_list:
            ranges = []
            for minimum, maximum in [self.state.ranges_viewport[axes.xaxis_index], self.state.ranges_viewport[axes.yaxis_index], ]:
                ranges.append(minimum)
                ranges.append(maximum)
            axes.rgb_images = []
            N = self.state.grid_size
            background = np.ones((N, N, 4), dtype=np.float64)
            background_color = self.background_color
            if background_color == "auto":
                if self.blend_mode in "screen lighten".split():
                    background_color = "black"
                else:
                    background_color = "white"
            background[:, :, 0:3] = matplotlib.colors.colorConverter.to_rgb(background_color)
            background[:, :, 3] = 1.
            axes.placeholder = axes.imshow(background, extent=ranges, origin="lower")
            axes.rgb_images.append(background)
            colors = "red green blue".split()
            axes.spines['bottom'].set_color(colors[axes.xaxis_index])
            axes.spines['left'].set_color(colors[axes.yaxis_index])
            linewidth = 2.
            axes.spines['bottom'].set_linewidth(linewidth)
            axes.spines['left'].set_linewidth(linewidth)
            if self.state.aspect is None:
                axes.set_aspect('auto')
            else:
                axes.set_aspect(self.state.aspect)
            axes.set_xlabel(self.get_label(axes.xaxis_index))
            axes.set_ylabel(self.get_label(axes.yaxis_index))

        for layer in self.layers:
            layer.plot(axes_list, self.add_image_layer)
        # for image in self.image_layers:
        if first_layer.amplitude_grid_selection is not None:
            self.widget_volume.setGrid(first_layer.amplitude_grid_selection.T, first_layer.amplitude_grid.T, first_layer.vector_grid)
        else:
            self.widget_volume.setGrid(first_layer.amplitude_grid.T, vectorgrid=first_layer.vector_grid)

        for axes in axes_list:
            rgba = vaex.ui.imageblending.blend(axes.rgb_images, self.blend_mode)
            rgba[..., 3] = rgba[..., 3] * 0 + 1
            for c in range(4):
                # rgba_dest[:,:,c] = np.clip((rgba_dest[:,:,c] ** 3.5)*2.6, 0., 1.)
                rgba[:, :, c] = np.clip((rgba[:, :, c] ** self.layer_gamma) * self.layer_brightness, 0., 1.)
            rgba = np.transpose(rgba, (1, 0, 2))
            axes.placeholder.set_data((rgba * 255).astype(np.uint8))

        # if self.state.aspect is None:
        # self.axes.set_aspect('auto')
        # else:
        # self.axes.set_aspect(self.state.aspect)
        # self.axes.set_xlim(*self.state.ranges_viewport[0])
        # self.axes.set_ylim(*self.state.ranges_viewport[1])
        self.fig.tight_layout()  # 1.008) #pad=pad, h_pad=h_pad, w_pad=w_pad, rect=rect)
        self.canvas.draw()
        self.update()

    def plot_(self):
        timelog("plot start", reset=False)
        t0 = time.time()
        if 1:
            ranges = []
            for minimum, maximum in self.state.ranges_viewport:
                ranges.append(minimum)
                ranges.append(maximum)

            timelog("creating grid map")
            grid_map = self.create_grid_map(self.state.grid_size, False)
            timelog("eval amplitude")
            amplitude = self.eval_amplitude(self.amplitude_expression, locals=grid_map)
            timelog("eval amplitude done")
            use_selection = self.dataset.mask is not None
            if use_selection:
                timelog("repeat for selection")
                grid_map_selection = self.create_grid_map(self.state.grid_size, True)
                amplitude_selection = self.eval_amplitude(self.amplitude_expression, locals=grid_map_selection)

            timelog("creating grid map vector")
            grid_map_vector = self.create_grid_map(self.state.vector_grid_size, use_selection)
            vector_grid = None
            vector_counts = grid_map_vector["counts"]
            vector_mask = vector_counts > 0
            if grid_map_vector["weightx"] is not None:
                vector_x = grid_map_vector["x"]
                vx = self.eval_amplitude("weightx/counts", locals=grid_map_vector)
            else:
                vector_x = None
                vx = None
            if grid_map_vector["weighty"] is not None:
                vector_y = grid_map_vector["y"]
                vy = self.eval_amplitude("weighty/counts", locals=grid_map_vector)
            else:
                vector_y = None
                vy = None
            if grid_map_vector["weightz"] is not None:
                vector_z = grid_map_vector["z"]
                vz = self.eval_amplitude("weightz/counts", locals=grid_map_vector)
            else:
                vector_z = None
                vz = None
            if vx is not None and vy is not None and vz is not None:
                timelog("making vector grid")
                vector_grid = np.zeros((4, ) + ((vx.shape[0],) * 3), dtype=np.float32)
                mask = vector_counts > 0
                meanvx = 0 if self.vectors_subtract_mean is False else vx[mask].mean()
                meanvy = 0 if self.vectors_subtract_mean is False else vy[mask].mean()
                meanvz = 0 if self.vectors_subtract_mean is False else vz[mask].mean()
                vector_grid[0] = vx - meanvx
                vector_grid[1] = vy - meanvy
                vector_grid[2] = vz - meanvz
                vector_grid[3] = vector_counts
                vector_grid = np.swapaxes(vector_grid, 0, 3)
                vector_grid = vector_grid * 1.
            timelog("setting grid")
            if use_selection:
                self.widget_volume.setGrid(amplitude_selection, amplitude, vectorgrid=vector_grid)
            else:
                self.widget_volume.setGrid(amplitude, vectorgrid=vector_grid)
            timelog("grid")
            if 0:
                self.tool.grid = amplitude
                self.tool.update()

            # if self.ranges_level[0] is None:
            # self.ranges_level[0] = 0, amplitude.max() * 1.1
            # return

            def multisum(a, axes):
                correction = 0
                for axis in axes:
                    a = np.nansum(a, axis=axis - correction)
                    correction += 1
                return a
            axeslist = self.getAxesList()
            vector_values = [vx, vy, vz]
            vector_positions = [vector_x, vector_y, vector_z]
            for i in range(2):
                timelog("axis: " + str(i))
                axes = axeslist[i]
                i1 = 0
                i2 = i + 1
                i3 = 2 - i
                ranges = list(self.ranges[i1]) + list(self.ranges[i2])
                axes.clear()
                allaxes = list(range(self.dimensions))
                if 0:  # i > 0:
                    for label in axes.get_yticklabels():
                        label.set_visible(False)
                    axes.yaxis.offsetText.set_visible(False)
                if 0:  # j > 0:
                    for label in axes.get_xticklabels():
                        label.set_visible(False)
                    axes.xaxis.offsetText.set_visible(False)
                if 1:
                    allaxes.remove(2 - (0))
                    allaxes.remove(2 - (1 + i))
                    counts_mask = None
                    colors = "red green blue".split()
                    axes.spines['bottom'].set_color(colors[i1])
                    axes.spines['left'].set_color(colors[i2])
                    linewidth = 2.
                    axes.spines['bottom'].set_linewidth(linewidth)
                    axes.spines['left'].set_linewidth(linewidth)

                    grid_map_2d = {key: None if grid is None else (grid if grid.ndim != 3 else multisum(grid, allaxes)) for key, grid in list(grid_map.items())}
                    amplitude = self.eval_amplitude(self.amplitude_expression, locals=grid_map_2d)
                    if use_selection:
                        grid_map_selection_2d = {key: None if grid is None else (grid if grid.ndim != 3 else multisum(grid, allaxes)) for key, grid in list(grid_map_selection.items())}
                        amplitude_selection = self.eval_amplitude(self.amplitude_expression, locals=grid_map_selection_2d)

                    axes.imshow(self.contrast(amplitude), origin="lower", extent=ranges, alpha=0.4 if use_selection else 1.0, cmap=self.colormap)
                    if use_selection:
                        axes.imshow(self.contrast(amplitude_selection), origin="lower", extent=ranges, alpha=1, cmap=self.colormap)

                    # vector_positions1, vector_positions2,  = vector_positions[i1],  vector_positions[i2]
                    # vector_values1, vector_values2 = vector_values[i1],  vector_values[i2]
                    if vector_positions[i1] is not None and vector_positions[i2] is not None:
                        mask = multisum(vector_counts, allaxes) > 0
                        x, y = np.meshgrid(vector_positions[i1], vector_positions[i2])
                        U = multisum(vector_values[i1], allaxes)
                        V = multisum(vector_values[i2], allaxes)

                        if np.any(mask):
                            meanU = 0 if self.vectors_subtract_mean is False else np.nanmean(U[mask])
                            meanV = 0 if self.vectors_subtract_mean is False else np.nanmean(V[mask])
                            U -= meanU
                            V -= meanV

                        if vector_positions[i3] is not None and self.vectors_color_code_3rd:
                            W = multisum(vector_values[i3], allaxes)
                            if np.any(mask):
                                meanW = 0 if self.vectors_subtract_mean is False else np.nanmean(W[mask])
                                W -= meanW
                            axes.quiver(x[mask], y[mask], U[mask], V[mask], W[mask], cmap=self.colormap_vector)
                        else:
                            axes.quiver(x[mask], y[mask], U[mask], V[mask], color="black")

                    if 0:  # TODO: self.dataset.selected_row_index is not None:
                        # self.axes.autoscale(False)
                        x, y = self.getdatax()[self.dataset.selected_row_index], self.getdatay()[self.dataset.selected_row_index]
                        axes.scatter([x], [y], color='red')  # , scalex=False, scaley=False)
                if self.state.aspect is None:
                    axes.set_aspect('auto')
                else:
                    axes.set_aspect(self.state.aspect)

                axes.set_xlim(self.state.ranges_viewport[i1][0], self.state.ranges_viewport[i1][1])
                axes.set_ylim(self.state.ranges_viewport[i2][0], self.state.ranges_viewport[i2][1])
                axes.set_xlabel(self.expressions[i1])
                axes.set_ylabel(self.expressions[i2])
            if 0:

                self.axes.imshow(amplitude.T, origin="lower", extent=ranges, alpha=1 if self.counts_mask is None else 0.4, cmap=cm_plusmin)
                if 1:
                    if self.counts_mask is not None:
                        if self.amplitude_expression is not None:
                            # locals = {"counts":self.counts_mask}
                            locals = {"counts": self.counts_weights_mask, "counts1": self.counts_mask}
                            globals = np.__dict__
                            amplitude_mask = eval(self.amplitude_expression, globals, locals)
                        self.axes.imshow(amplitude_mask.T, origin="lower", extent=ranges, alpha=1, cmap=cm_plusmin)
                    # self.axes.imshow((I), origin="lower", extent=ranges)
                self.axes.set_aspect('auto')
                # if self.dataset.selected_row_index is not None:
                # self.axes.autoscale(False)
                index = self.dataset.selected_row_index
                if index is not None and self.selected_point is None:
                    logger.debug("point selected but after computation")
                    # TODO: optimize

                    def find_selected_point(info, blockx, blocky):
                        if index >= info.i1 and index < info.i2:  # selected point is in this block
                            self.selected_point = blockx[index - info.i1], blocky[index - info.i1]
                    self.dataset.evaluate(find_selected_point, *self.expressions, **self.getVariableDict())

                if self.selected_point:
                    # x, y = self.getdatax()[self.dataset.selected_row_index],  self.getdatay()[self.dataset.selected_row_index]
                    x, y = self.selected_point
                    self.axes.scatter([x], [y], color='red')  # , scalex=False, scaley=False)
                # if dataxsel is not None:
                # self.axes.scatter(dataxsel, dataysel)
                self.axes.set_xlabel(self.expressions[0])
                self.axes.set_ylabel(self.expressions[0])
                self.axes.set_xlim(*self.state.ranges_viewport[0])
                self.axes.set_ylim(*self.state.ranges_viewport[1])
        self.canvas.draw()
        timelog("plot end")
        self.message("ploting %f" % (time.time() - t0), index=5)

class Rank1ScatterPlotDialog(ScatterPlotDialog):
    type_name = "sequence-density2d"

    def __init__(self, parent, dataset, xname=None, yname=None):
        self.nSlices = dataset.rank1s[list(dataset.rank1s.keys())[0]].shape[0]
        self.serieIndex = dataset.selected_serie_index if dataset.selected_serie_index is not None else 0
        self.record_frames = False
        super(Rank1ScatterPlotDialog, self).__init__(parent, dataset, xname, yname)

    def getTitleExpressionList(self):
        # return []
        return ["%s: {%s: 4f}" % (name, name) for name in self.dataset.axis_names]

    def addToolbar2(self, layout, contrast=True, gamma=True):
        super(Rank1ScatterPlotDialog, self).addToolbar2(layout, contrast, gamma)
        self.action_save_frames = QtGui.QAction(QtGui.QIcon(iconfile('film')), '&Export frames', self)
        self.menu_save.addAction(self.action_save_frames)
        self.action_save_frames.triggered.connect(self.onActionSaveFrames)

    def onActionSaveFrames(self, ignore=None):
        # directory = QtGui.QFileDialog.getExistingDirectory(self, "Choose where to save frames", "",  QtGui.QFileDialog.ShowDirsOnly | QtGui.QFileDialog.DontResolveSymlinks)
        directory = qt.getdir(self, "Choose where to save frames", "")
        self.frame_template = os.path.join(directory, "%s_{index:05}.png" % self.dataset.name)
        self.frame_template = qt.gettext(self, "template for frame filenames", "template:", self.frame_template)
        self.record_frames = True
        self.onPlayOnce()

    def plot(self):
        super(Rank1ScatterPlotDialog, self).plot()
        if self.record_frames:
            index = self.serieIndex
            path = self.frame_template.format(**locals())
            self.fig.savefig(path)
            if self.serieIndex == self.nSlices - 1:
                self.record_frames = False

    def onSerieIndexSelect(self, serie_index):
        if serie_index != self.serieIndex:  # avoid unneeded event
            self.serieIndex = serie_index
            self.seriesbox.setCurrentIndex(self.serieIndex)
        else:
            self.serieIndex = serie_index
        self.compute()
        # self.jobsM
        # self.plot()

    def getExpressionList(self):
        names = []
        for rank1name in self.dataset.rank1names:
            names.append(rank1name + "[index]")
        return names

    def getVariableDict(self):
        vars = {"index": self.serieIndex}
        for name in self.dataset.axis_names:
            vars[name] = self.dataset.axes[name][self.serieIndex]
        return vars

    def _getVariableDictMinMax(self):
        return {"index": slice(None, None, None)}

    def afterCanvas(self, layout):
        super(Rank1ScatterPlotDialog, self).afterCanvas(layout)
        # return

        self.seriesbox = QtGui.QComboBox(self)
        self.seriesbox.addItems([str(k) for k in range(self.nSlices)])
        self.seriesbox.setCurrentIndex(self.serieIndex)
        self.seriesbox.currentIndexChanged.connect(self.onSerieIndex)

        self.grid_layout.addWidget(self.seriesbox, 10, 1)
        # self.form_layout = QtGui.QFormLayout(self)
        # self.form_layout.addRow("index", self.seriesbox)
        # self.buttonLoop = QtGui.QToolButton(self)
        # self.buttonLoop.setText("one loop")
        # self.buttonLoop.clicked.connect(self.onPlayOnce)
        # self.form_layout.addRow("movie", self.buttonLoop)
        # layout.addLayout(self.form_layout, 0)

    def onPlayOnce(self):
        # self.timer = QtCore.QTimer(self)
        # self.timer.timeout.connect(self.onNextFrame)
        self.delay = 10
        if not self.state.axis_lock:
            for i in range(self.dimensions):
                self.ranges[i] = None
            for i in range(self.dimensions):
                self.state.ranges_viewport[i] = None
        self.dataset.selectSerieIndex(0)
        self.dataset.executor.execute()
        QtCore.QTimer.singleShot(self.delay if not self.record_frames else 0, self.onNextFrame)

    def onNextFrame(self, *args):
        step = 1
        next = self.serieIndex + step
        if next >= self.nSlices:
            next = self.nSlices - 1
        if not self.state.axis_lock:
            for i in range(self.dimensions):
                self.ranges[i] = None
            for i in range(self.dimensions):
                self.state.ranges_viewport[i] = None
        self.dataset.selectSerieIndex(next)
        self.dataset.executor.execute()
        if self.serieIndex < self.nSlices - 1:  # not last frame
            QtCore.QTimer.singleShot(self.delay, self.onNextFrame)

    def onSerieIndex(self, index):
        if index != self.dataset.selected_serie_index:  # avoid unneeded event
            if not self.state.axis_lock:
                for i in range(self.dimensions):
                    self.ranges[i] = None
                for i in range(self.dimensions):
                    self.state.ranges_viewport[i] = None
            self.dataset.selectSerieIndex(index)
            # self.compute()
            self.dataset.executor.execute()

class Mover(object):
    def __init__(self, plot, axes):
        self.plot = plot
        self.axes = axes
        self.canvas = self.axes.figure.canvas
        self.axes = None

        self.canvas.mpl_connect('scroll_event', self.mpl_scroll)
        self.last_x, self.last_y = None, None
        self.handles = []
        self.handles.append(self.canvas.mpl_connect('motion_notify_event', self.mouse_move))
        self.handles.append(self.canvas.mpl_connect('button_press_event', self.mouse_down))
        self.handles.append(self.canvas.mpl_connect('button_release_event', self.mouse_up))
        self.begin_x, self.begin_y = None, None
        self.moved = False
        self.zoom_queue = []
        self.zoom_counter = 0

    def disconnect_events(self):
        for handle in self.handles:
            self.canvas.mpl_disconnect(handle)

    def mouse_up(self, event):
        self.last_x, self.last_y = None, None
        if self.moved:
            # self.plot.ranges = list(self.plot.state.ranges_viewport)
            # for layer in self.plot.layers:
            # layer.ranges = list(self.plot)
            # self.plot.compute()
            # self.plot.jobsManager.execute()
            self.plot.queue_history_change("pan")
            self.plot.update_all_layers()
            self.moved = False

    def mouse_down(self, event):
        self.moved = False
        if event.dblclick:
            factor = 0.333
            if event.button != 1:
                factor = 1 / factor
            self.plot.zoom(factor, axes=event.inaxes, x=event.xdata, y=event.ydata)
            self.plot.queue_history_change("zoom in around (%s, %s)" % (event.xdata, event.ydata))
        else:
            self.begin_x, self.begin_y = event.xdata, event.ydata
            self.last_x, self.last_y = event.xdata, event.ydata
            self.current_axes = event.inaxes
            self.plot.ranges_begin = list(self.plot.state.ranges_viewport)

    def mouse_move(self, event):
        # return
        if self.last_x is not None and event.xdata is not None and self.current_axes is not None:
            # axes = event.inaxes
            transform = self.current_axes.transData.inverted().transform
            x_data, y_data = event.xdata, event.ydata
            self.moved = True
            dx = self.last_x - x_data
            dy = self.last_y - y_data
            if self.plot.state.ranges_viewport is None or None in self.plot.state.ranges_viewport:
                return  # this means we didn't even have a viewport, and already want to zoom/pan
            xmin, xmax = self.plot.state.ranges_viewport[self.current_axes.xaxis_index][0] + dx, self.plot.state.ranges_viewport[self.current_axes.xaxis_index][1] + dx
            if self.plot.dimensions == 1:
                ymin, ymax = self.plot.state.range_level_show[0] + dy, self.plot.state.range_level_show[1] + dy
            else:
                ymin, ymax = self.plot.state.ranges_viewport[self.current_axes.yaxis_index][0] + dy, self.plot.state.ranges_viewport[self.current_axes.yaxis_index][1] + dy
            # self.plot.state.ranges_viewport = [[xmin, xmax], [ymin, ymax]]
            self.plot.state.ranges_viewport[self.current_axes.xaxis_index] = [xmin, xmax]
            if self.plot.dimensions == 1:
                self.plot.state.range_level_show = [ymin, ymax]
            else:
                self.plot.state.ranges_viewport[self.current_axes.yaxis_index] = [ymin, ymax]
            # TODO: maybe the dimension should be stored in the axes, not in the plotdialog
            for axes in self.plot.getAxesList():
                if self.plot.dimensions == 1:
                    # ftm we assume we only have 1 histogram, meabning axes == self.current_axes
                    axes.set_xlim(*self.plot.state.ranges_viewport[self.current_axes.xaxis_index])
                    axes.set_ylim(*self.plot.state.range_level_show)
                else:
                    if axes.xaxis_index == self.current_axes.xaxis_index:
                        axes.set_xlim(*self.plot.state.ranges_viewport[self.current_axes.xaxis_index])
                    if axes.yaxis_index == self.current_axes.xaxis_index:
                        axes.set_ylim(*self.plot.state.ranges_viewport[self.current_axes.xaxis_index])
                    if axes.xaxis_index == self.current_axes.yaxis_index:
                        axes.set_xlim(*self.plot.state.ranges_viewport[self.current_axes.yaxis_index])
                    if axes.yaxis_index == self.current_axes.yaxis_index:
                        axes.set_ylim(*self.plot.state.ranges_viewport[self.current_axes.yaxis_index])

            # transform again after we changed the axes limits
            transform = self.current_axes.transData.inverted().transform
            x_data, y_data = transform([event.x * 1., event.y * 1])
            self.last_x, self.last_y = x_data, y_data

            self.canvas.draw_idle()

    def mpl_scroll(self, event):
        factor = 10**(-event.step / 8)
        self.zoom_counter += 1
        if event.inaxes is None:
            return

        if factor < 1:
            self.plot.zoom(factor, event.inaxes, event.xdata, event.ydata)
            # self.zoom_queue.append((factor, event.xdata, event.ydata))
        else:
            self.plot.zoom(factor, event.inaxes, event.xdata, event.ydata)  # , event.xdata, event.ydata)
            # self.zoom_queue.append((factor, None, None))
        return

        def idle_zoom(ignore=None, zoom_counter=None, axes=None):

            if zoom_counter < self.zoom_counter:
                pass  # ignore, a later event will come
            else:
                # zoom_queue = list((self.zoom_queue) # make copy to ensure it doesn't get modified in
                for i, (factor, x, y) in enumerate(self.zoom_queue):
                    # only redraw at last call
                    is_last = i == len(self.zoom_queue) - 1
                    self.plot.zoom(factor, axes=axes, x=x, y=y, recalculate=False, history=False, redraw=is_last)
                self.zoom_queue = []
        if event.axes:
            QtCore.QTimer.singleShot(1, functools.partial(idle_zoom, zoom_counter=self.zoom_counter, axes=event.inaxes))

class Queue(object):
    logger = logging.getLogger("vaex.ui.queue")

    def __init__(self, name, default_delay, default_callable, pre=lambda: None):
        self.name = name
        self.default_delay = default_delay
        self.counter = 0
        self.counter_processed = 0
        self.default_callable = default_callable
        self.pre = pre

    def is_empty(self):
        return (self.counter_processed == self.counter)

    def in_queue(self, minimum=1):
        return (self.counter_processed <= (self.counter - minimum))

    def _wait(self, sleep=10):
        if self.counter > 0:
            qt_app = QtCore.QCoreApplication.instance()
            logger.debug("*** waiting for queue %r" % self.name)
            while self.counter_processed != self.counter:
                logger.debug("counter=%i counter_processed=%i", self.counter, self.counter_processed)
                qt_app.processEvents()
                QtTest.QTest.qSleep(sleep)
            logger.debug("*** done with queue %r" % self.name)

    def cancel(self):
        def nop():
            self.logger.debug("nop called in queue %r" % self.name)
        self.logger.debug("cancelling last call in queue %r by inserting a nop" % self.name)
        self(nop)

    def __call__(self, callable=None, delay=None, *args, **kwargs):
        self.pre()
        # print self.name
        # import traceback
        # traceback.print_stack()
        if delay is None:
            delay = self.default_delay
        callable = callable or self.default_callable

        def call(_=None, counter=None, callable=None):
            try:
                if counter < self.counter:
                    pass  # ignore, more events coming
                    self.logger.debug("ignoring this event in queue %r, since a new one is scheduled" % self.name)
                else:
                    self.logger.debug("calling callback in queue %r" % self.name)
                    callable()
                    self.counter_processed = self.counter - 1  # to make the queue 'empty'
            finally:
                self.counter_processed = counter
        callable = functools.partial(callable, *args, **kwargs)
        self.counter += 1
        self.logger.debug("add in queue %r %r %s" % (self.name, delay, threading.currentThread()))
        # traceback.print_stack()
        # import traceback
        # traceback.print_stack()
        if delay == 0:
            call(counter=self.counter, callable=callable)
        else:
            QtCore.QTimer.singleShot(delay, functools.partial(call, counter=self.counter, callable=callable))

from vaex.ui.layers import LayerTable
import vaex.ui.layers
try:
    from vaex.ui.main import VaexApp
except:
    pass