def clear(self, event):
        """clear the cursor"""
        if self.useblit:
            self.background = (
                self.canvas.copy_from_bbox(self.canvas.figure.bbox))
        for line in self.vlines + self.hlines:
            line.set_visible(False)
        self.ellipse.set_visible(False)

def clear ( self , event ) : if self . useblit : self . background = ( self . canvas . copy_from_bbox ( self . canvas . figure . bbox ) ) for line in self . vlines + self . hlines : line . set_visible ( False ) self . ellipse . set_visible ( False )


def _wait(self):
        """Used for unittesting to make sure the plots are all done"""
        logger.debug("will wait for last plot to finish")
        self._plot_event = threading.Event()
        self.queue_update._wait()
        self.queue_replot._wait()
        self.queue_redraw._wait()
        qt_app = QtCore.QCoreApplication.instance()
        sleep = 10
        while not self._plot_event.is_set():
            logger.debug("waiting for last plot to finish")
            qt_app.processEvents()
            QtTest.QTest.qSleep(sleep)
        logger.debug("waiting for plot finished")

def
_wait
(
self
)
:
logger
.
debug
(
"will wait for last plot to finish"
)
self
.
_plot_event
=
threading
.
Event
(
)
self
.
queue_update
.
_wait
(
)
self
.
queue_replot
.
_wait
(
)
self
.
queue_redraw
.
_wait
(
)
qt_app
=
QtCore
.
QCoreApplication
.
instance
(
)
sleep
=
10
while
not
self
.
_plot_event
.
is_set
(
)
:
logger
.
debug
(
"waiting for last plot to finish"
)
qt_app
.
processEvents
(
)
QtTest
.
QTest
.
qSleep
(
sleep
)
logger
.
debug
(
"waiting for plot finished"
)


def os_open(document):
    """Open document by the default handler of the OS, could be a url opened by a browser, a text file by an editor etc"""
    osname = platform.system().lower()
    if osname == "darwin":
        os.system("open \"" + document + "\"")
    if osname == "linux":
        cmd = "xdg-open \"" + document + "\"&"
        os.system(cmd)
    if osname == "windows":
        os.system("start \"" + document + "\"")

def
os_open
(
document
)
:
osname
=
platform
.
system
(
)
.
lower
(
)
if
osname
==
"darwin"
:
os
.
system
(
"open \""
+
document
+
"\""
)
if
osname
==
"linux"
:
cmd
=
"xdg-open \""
+
document
+
"\"&"
os
.
system
(
cmd
)
if
osname
==
"windows"
:
os
.
system
(
"start \""
+
document
+
"\""
)

Open document by the default handler of the OS could be a url opened by a browser a text file by an editor etc


def write_to(f, mode):
    """Flexible writing, where f can be a filename or f object, if filename, closed after writing"""
    if hasattr(f, 'write'):
        yield f
    else:
        f = open(f, mode)
        yield f
        f.close()

def write_to ( f , mode ) : if hasattr ( f , 'write' ) : yield f else : f = open ( f , mode ) yield f f . close ( )

Flexible writing where f can be a filename or f object if filename closed after writing


def _split_and_combine_mask(arrays):
	'''Combines all masks from a list of arrays, and logically ors them into a single mask'''
	masks = [np.ma.getmaskarray(block) for block in arrays if np.ma.isMaskedArray(block)]
	arrays = [block.data if np.ma.isMaskedArray(block) else block for block in arrays]
	mask = None
	if masks:
		mask = masks[0].copy()
		for other in masks[1:]:
			mask |= other
	return arrays, mask

def _split_and_combine_mask ( arrays ) : masks = [ np . ma . getmaskarray ( block ) for block in arrays if np . ma . isMaskedArray ( block ) ] arrays = [ block . data if np . ma . isMaskedArray ( block ) else block for block in arrays ] mask = None if masks : mask = masks [ 0 ] . copy ( ) for other in masks [ 1 : ] : mask |= other return arrays , mask

Combines all masks from a list of arrays and logically ors them into a single mask


def nop(self, expression, progress=False, delay=False):
        """Evaluates expression, and drop the result, usefull for benchmarking, since vaex is usually lazy"""
        expression = _ensure_string_from_expression(expression)
        def map(ar):
            pass
        def reduce(a, b):
            pass
        return self.map_reduce(map, reduce, [expression], delay=delay, progress=progress, name='nop', to_numpy=False)

def nop ( self , expression , progress = False , delay = False ) : expression = _ensure_string_from_expression ( expression ) def map ( ar ) : pass def reduce ( a , b ) : pass return self . map_reduce ( map , reduce , [ expression ] , delay = delay , progress = progress , name = 'nop' , to_numpy = False )

Evaluates expression and drop the result usefull for benchmarking since vaex is usually lazy


def first(self, expression, order_expression, binby=[], limits=None, shape=default_shape, selection=False, delay=False, edges=False, progress=None):
        """Return the first element of a binned `expression`, where the values each bin are sorted by `order_expression`.

        Example:

        >>> import vaex
        >>> df = vaex.example()
        >>> df.first(df.x, df.y, shape=8)
        >>> df.first(df.x, df.y, shape=8, binby=[df.y])
        >>> df.first(df.x, df.y, shape=8, binby=[df.y])
        array([-4.81883764, 11.65378   ,  9.70084476, -7.3025589 ,  4.84954977,
                8.47446537, -5.73602629, 10.18783   ])

        :param expression: The value to be placed in the bin.
        :param order_expression: Order the values in the bins by this expression.
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :param edges: {edges}
        :return: Ndarray containing the first elements.
        :rtype: numpy.array
        """
        return self._compute_agg('first', expression, binby, limits, shape, selection, delay, edges, progress, extra_expressions=[order_expression])
        logger.debug("count(%r, binby=%r, limits=%r)", expression, binby, limits)
        logger.debug("count(%r, binby=%r, limits=%r)", expression, binby, limits)
        expression = _ensure_strings_from_expressions(expression)
        order_expression = _ensure_string_from_expression(order_expression)
        binby = _ensure_strings_from_expressions(binby)
        waslist, [expressions,] = vaex.utils.listify(expression)
        @delayed
        def finish(*counts):
            counts = np.asarray(counts)
            return vaex.utils.unlistify(waslist, counts)
        progressbar = vaex.utils.progressbars(progress)
        limits = self.limits(binby, limits, delay=True, shape=shape)
        stats = [self._first_calculation(expression, order_expression, binby=binby, limits=limits, shape=shape, selection=selection, edges=edges, progressbar=progressbar) for expression in expressions]
        var = finish(*stats)
        return self._delay(delay, var)

def
first
(
self
,
expression
,
order_expression
,
binby
=
[
]
,
limits
=
,
shape
=
default_shape
,
selection
=
False
,
delay
=
False
,
edges
=
False
,
progress
=
)
:
return
self
.
_compute_agg
(
'first'
,
expression
,
binby
,
limits
,
shape
,
selection
,
delay
,
edges
,
progress
,
extra_expressions
=
[
order_expression
]
)
logger
.
debug
(
"count(%r, binby=%r, limits=%r)"
,
expression
,
binby
,
limits
)
logger
.
debug
(
"count(%r, binby=%r, limits=%r)"
,
expression
,
binby
,
limits
)
expression
=
_ensure_strings_from_expressions
(
expression
)
order_expression
=
_ensure_string_from_expression
(
order_expression
)
binby
=
_ensure_strings_from_expressions
(
binby
)
waslist
,
[
expressions
,
]
=
vaex
.
utils
.
listify
(
expression
)
@
delayed
def
finish
(
*
counts
)
:
counts
=
np
.
asarray
(
counts
)
return
vaex
.
utils
.
unlistify
(
waslist
,
counts
)
progressbar
=
vaex
.
utils
.
progressbars
(
progress
)
limits
=
self
.
limits
(
binby
,
limits
,
delay
=
True
,
shape
=
shape
)
stats
=
[
self
.
_first_calculation
(
expression
,
order_expression
,
binby
=
binby
,
limits
=
limits
,
shape
=
shape
,
selection
=
selection
,
edges
=
edges
,
progressbar
=
progressbar
)
for
expression
in
expressions
]
var
=
finish
(
*
stats
)
return
self
.
_delay
(
delay
,
var
)

Return the first element of a binned expression where the values each bin are sorted by order_expression .


def mean(self, expression, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None, edges=False):
        """Calculate the mean for expression, possibly on a grid defined by binby.

        Example:

        >>> df.mean("x")
        -0.067131491264005971
        >>> df.mean("(x**2+y**2)**0.5", binby="E", shape=4)
        array([  2.43483742,   4.41840721,   8.26742458,  15.53846476])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :return: {return_stat_scalar}
        """
        return self._compute_agg('mean', expression, binby, limits, shape, selection, delay, edges, progress)
        logger.debug("mean of %r, with binby=%r, limits=%r, shape=%r, selection=%r, delay=%r", expression, binby, limits, shape, selection, delay)
        expression = _ensure_strings_from_expressions(expression)
        selection = _ensure_strings_from_expressions(selection)
        binby = _ensure_strings_from_expressions(binby)

        @delayed
        def calculate(expression, limits):
            task = tasks.TaskStatistic(self, binby, shape, limits, weight=expression, op=tasks.OP_ADD_WEIGHT_MOMENTS_01, selection=selection)
            self.executor.schedule(task)
            progressbar.add_task(task, "mean for %s" % expression)
            return task

        @delayed
        def finish(*stats_args):
            stats = np.array(stats_args)
            counts = stats[..., 0]
            with np.errstate(divide='ignore', invalid='ignore'):
                mean = stats[..., 1] / counts
            return vaex.utils.unlistify(waslist, mean)
        waslist, [expressions, ] = vaex.utils.listify(expression)
        progressbar = vaex.utils.progressbars(progress)
        limits = self.limits(binby, limits, delay=True)
        stats = [calculate(expression, limits) for expression in expressions]
        var = finish(*stats)
        return self._delay(delay, var)

Calculate the mean for expression possibly on a grid defined by binby .


def sum(self, expression, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None, edges=False):
        """Calculate the sum for the given expression, possible on a grid defined by binby

        Example:

        >>> df.sum("L")
        304054882.49378014
        >>> df.sum("L", binby="E", shape=4)
        array([  8.83517994e+06,   5.92217598e+07,   9.55218726e+07,
                         1.40008776e+08])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :return: {return_stat_scalar}
        """
        return self._compute_agg('sum', expression, binby, limits, shape, selection, delay, edges, progress)
        @delayed
        def finish(*sums):
            return vaex.utils.unlistify(waslist, sums)
        expression = _ensure_strings_from_expressions(expression)
        binby = _ensure_strings_from_expressions(binby)
        waslist, [expressions, ] = vaex.utils.listify(expression)
        progressbar = vaex.utils.progressbars(progress)
        limits = self.limits(binby, limits, delay=True)
        # stats = [calculate(expression, limits) for expression in expressions]
        sums = [self._sum_calculation(expression, binby=binby, limits=limits, shape=shape, selection=selection, progressbar=progressbar) for expression in expressions]
        s = finish(*sums)
        return self._delay(delay, s)

Calculate the sum for the given expression possible on a grid defined by binby


def std(self, expression, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):
        """Calculate the standard deviation for the given expression, possible on a grid defined by binby

        >>> df.std("vz")
        110.31773397535071
        >>> df.std("vz", binby=["(x**2+y**2)**0.5"], shape=4)
        array([ 123.57954851,   85.35190177,   61.14345748,   38.0740619 ])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :return: {return_stat_scalar}
        """
        @delayed
        def finish(var):
            return var**0.5
        return self._delay(delay, finish(self.var(expression, binby=binby, limits=limits, shape=shape, selection=selection, delay=True, progress=progress)))

def std ( self , expression , binby = [ ] , limits = None , shape = default_shape , selection = False , delay = False , progress = None ) : @ delayed def finish ( var ) : return var ** 0.5 return self . _delay ( delay , finish ( self . var ( expression , binby = binby , limits = limits , shape = shape , selection = selection , delay = True , progress = progress ) ) )

Calculate the standard deviation for the given expression possible on a grid defined by binby


def cov(self, x, y=None, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):
        """Calculate the covariance matrix for x and y or more expressions, possibly on a grid defined by binby.

        Either x and y are expressions, e.g:

        >>> df.cov("x", "y")

        Or only the x argument is given with a list of expressions, e,g.:

        >>> df.cov(["x, "y, "z"])

        Example:

        >>> df.cov("x", "y")
        array([[ 53.54521742,  -3.8123135 ],
        [ -3.8123135 ,  60.62257881]])
        >>> df.cov(["x", "y", "z"])
        array([[ 53.54521742,  -3.8123135 ,  -0.98260511],
        [ -3.8123135 ,  60.62257881,   1.21381057],
        [ -0.98260511,   1.21381057,  25.55517638]])

        >>> df.cov("x", "y", binby="E", shape=2)
        array([[[  9.74852878e+00,  -3.02004780e-02],
        [ -3.02004780e-02,   9.99288215e+00]],
        [[  8.43996546e+01,  -6.51984181e+00],
        [ -6.51984181e+00,   9.68938284e+01]]])

        :param x: {expression}
        :param y: {expression_single}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :return: {return_stat_scalar}, the last dimensions are of shape (2,2)
        """
        selection = _ensure_strings_from_expressions(selection)
        if y is None:
            if not _issequence(x):
                raise ValueError("if y argument is not given, x is expected to be sequence, not %r", x)
            expressions = x
        else:
            expressions = [x, y]
        N = len(expressions)
        binby = _ensure_list(binby)
        shape = _expand_shape(shape, len(binby))
        progressbar = vaex.utils.progressbars(progress)
        limits = self.limits(binby, limits, selection=selection, delay=True)

        @delayed
        def calculate(expressions, limits):
            # print('limits', limits)
            task = tasks.TaskStatistic(self, binby, shape, limits, weights=expressions, op=tasks.OP_COV, selection=selection)
            self.executor.schedule(task)
            progressbar.add_task(task, "covariance values for %r" % expressions)
            return task

        @delayed
        def finish(values):
            N = len(expressions)
            counts = values[..., :N]
            sums = values[..., N:2 * N]
            with np.errstate(divide='ignore', invalid='ignore'):
                means = sums / counts
            # matrix of means * means.T
            meansxy = means[..., None] * means[..., None, :]

            counts = values[..., 2 * N:2 * N + N**2]
            sums = values[..., 2 * N + N**2:]
            shape = counts.shape[:-1] + (N, N)
            counts = counts.reshape(shape)
            sums = sums.reshape(shape)
            with np.errstate(divide='ignore', invalid='ignore'):
                moments2 = sums / counts
            cov_matrix = moments2 - meansxy
            return cov_matrix
        progressbar = vaex.utils.progressbars(progress)
        values = calculate(expressions, limits)
        cov_matrix = finish(values)
        return self._delay(delay, cov_matrix)

Calculate the covariance matrix for x and y or more expressions possibly on a grid defined by binby .


def minmax(self, expression, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):
        """Calculate the minimum and maximum for expressions, possibly on a grid defined by binby.

        Example:

        >>> df.minmax("x")
        array([-128.293991,  271.365997])
        >>> df.minmax(["x", "y"])
        array([[-128.293991 ,  271.365997 ],
                   [ -71.5523682,  146.465836 ]])
        >>> df.minmax("x", binby="x", shape=5, limits=[-10, 10])
        array([[-9.99919128, -6.00010443],
                   [-5.99972439, -2.00002384],
                   [-1.99991322,  1.99998057],
                   [ 2.0000093 ,  5.99983597],
                   [ 6.0004878 ,  9.99984646]])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :return: {return_stat_scalar}, the last dimension is of shape (2)
        """
        # vmin  = self._compute_agg('min', expression, binby, limits, shape, selection, delay, edges, progress)
        # vmax =  self._compute_agg('max', expression, binby, limits, shape, selection, delay, edges, progress)
        @delayed
        def finish(*minmax_list):
            value = vaex.utils.unlistify(waslist, np.array(minmax_list))
            value = value.astype(dtype0)
            return value

        @delayed
        def calculate(expression, limits):
            task = tasks.TaskStatistic(self, binby, shape, limits, weight=expression, op=tasks.OP_MIN_MAX, selection=selection)
            self.executor.schedule(task)
            progressbar.add_task(task, "minmax for %s" % expression)
            return task
        @delayed
        def finish(*minmax_list):
            value = vaex.utils.unlistify(waslist, np.array(minmax_list))
            value = value.astype(dtype0)
            return value
        expression = _ensure_strings_from_expressions(expression)
        binby = _ensure_strings_from_expressions(binby)
        waslist, [expressions, ] = vaex.utils.listify(expression)
        dtypes = [self.dtype(expr) for expr in expressions]
        dtype0 = dtypes[0]
        if not all([k.kind == dtype0.kind for k in dtypes]):
            raise ValueError("cannot mix datetime and non-datetime expressions")
        progressbar = vaex.utils.progressbars(progress, name="minmaxes")
        limits = self.limits(binby, limits, selection=selection, delay=True)
        all_tasks = [calculate(expression, limits) for expression in expressions]
        result = finish(*all_tasks)
        return self._delay(delay, result)

Calculate the minimum and maximum for expressions possibly on a grid defined by binby .


def min(self, expression, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None, edges=False):
        """Calculate the minimum for given expressions, possibly on a grid defined by binby.

        Example:

        >>> df.min("x")
        array(-128.293991)
        >>> df.min(["x", "y"])
        array([-128.293991 ,  -71.5523682])
        >>> df.min("x", binby="x", shape=5, limits=[-10, 10])
        array([-9.99919128, -5.99972439, -1.99991322,  2.0000093 ,  6.0004878 ])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :return: {return_stat_scalar}, the last dimension is of shape (2)
        """
        return self._compute_agg('min', expression, binby, limits, shape, selection, delay, edges, progress)
        @delayed
        def finish(result):
            return result[..., 0]
        return self._delay(delay, finish(self.minmax(expression, binby=binby, limits=limits, shape=shape, selection=selection, delay=delay, progress=progress)))

def min ( self , expression , binby = [ ] , limits = None , shape = default_shape , selection = False , delay = False , progress = None , edges = False ) : return self . _compute_agg ( 'min' , expression , binby , limits , shape , selection , delay , edges , progress ) @ delayed def finish ( result ) : return result [ ... , 0 ] return self . _delay ( delay , finish ( self . minmax ( expression , binby = binby , limits = limits , shape = shape , selection = selection , delay = delay , progress = progress ) ) )

Calculate the minimum for given expressions possibly on a grid defined by binby .


def median_approx(self, expression, percentage=50., binby=[], limits=None, shape=default_shape, percentile_shape=256, percentile_limits="minmax", selection=False, delay=False):
        """Calculate the median , possibly on a grid defined by binby.

        NOTE: this value is approximated by calculating the cumulative distribution on a grid defined by
        percentile_shape and percentile_limits

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param percentile_limits: {percentile_limits}
        :param percentile_shape: {percentile_shape}
        :param selection: {selection}
        :param delay: {delay}
        :return: {return_stat_scalar}
        """
        return self.percentile_approx(expression, 50, binby=binby, limits=limits, shape=shape, percentile_shape=percentile_shape, percentile_limits=percentile_limits, selection=selection, delay=delay)

def median_approx ( self , expression , percentage = 50. , binby = [ ] , limits = None , shape = default_shape , percentile_shape = 256 , percentile_limits = "minmax" , selection = False , delay = False ) : return self . percentile_approx ( expression , 50 , binby = binby , limits = limits , shape = shape , percentile_shape = percentile_shape , percentile_limits = percentile_limits , selection = selection , delay = delay )

Calculate the median possibly on a grid defined by binby .


def plot_widget(self, x, y, z=None, grid=None, shape=256, limits=None, what="count(*)", figsize=None,
                    f="identity", figure_key=None, fig=None, axes=None, xlabel=None, ylabel=None, title=None,
                    show=True, selection=[None, True], colormap="afmhot", grid_limits=None, normalize="normalize",
                    grid_before=None,
                    what_kwargs={}, type="default",
                    scales=None, tool_select=False, bq_cleanup=True,
                    backend="bqplot",
                    **kwargs):
        """Viz 1d, 2d or 3d in a Jupyter notebook

        .. note::
            This API is not fully settled and may change in the future

        Example:

        >>> df.plot_widget(df.x, df.y, backend='bqplot')
        >>> df.plot_widget(df.pickup_longitude, df.pickup_latitude, backend='ipyleaflet')

        :param backend: Widget backend to use: 'bqplot', 'ipyleaflet', 'ipyvolume', 'matplotlib'

        """
        import vaex.jupyter.plot
        backend = vaex.jupyter.plot.create_backend(backend)
        cls = vaex.jupyter.plot.get_type(type)
        x = _ensure_strings_from_expressions(x)
        y = _ensure_strings_from_expressions(y)
        z = _ensure_strings_from_expressions(z)
        for name in 'vx vy vz'.split():
            if name in kwargs:
                kwargs[name] = _ensure_strings_from_expressions(kwargs[name])
        plot2d = cls(backend=backend, dataset=self, x=x, y=y, z=z, grid=grid, shape=shape, limits=limits, what=what,
                     f=f, figure_key=figure_key, fig=fig,
                     selection=selection, grid_before=grid_before,
                     grid_limits=grid_limits, normalize=normalize, colormap=colormap, what_kwargs=what_kwargs, **kwargs)
        if show:
            plot2d.show()
        return plot2d

def
plot_widget
(
self
,
x
,
y
,
z
=
,
grid
=
,
shape
=
256
,
limits
=
,
what
=
"count(*)"
,
figsize
=
,
f
=
"identity"
,
figure_key
=
,
fig
=
,
axes
=
,
xlabel
=
,
ylabel
=
,
title
=
,
show
=
True
,
selection
=
[
,
True
]
,
colormap
=
"afmhot"
,
grid_limits
=
,
normalize
=
"normalize"
,
grid_before
=
,
what_kwargs
=
{
}
,
type
=
"default"
,
scales
=
,
tool_select
=
False
,
bq_cleanup
=
True
,
backend
=
"bqplot"
,
*
*
kwargs
)
:
import
vaex
.
jupyter
.
plot
backend
=
vaex
.
jupyter
.
plot
.
create_backend
(
backend
)
cls
=
vaex
.
jupyter
.
plot
.
get_type
(
type
)
x
=
_ensure_strings_from_expressions
(
x
)
y
=
_ensure_strings_from_expressions
(
y
)
z
=
_ensure_strings_from_expressions
(
z
)
for
name
in
'vx vy vz'
.
split
(
)
:
if
name
in
kwargs
:
kwargs
[
name
]
=
_ensure_strings_from_expressions
(
kwargs
[
name
]
)
plot2d
=
cls
(
backend
=
backend
,
dataset
=
self
,
x
=
x
,
y
=
y
,
z
=
z
,
grid
=
grid
,
shape
=
shape
,
limits
=
limits
,
what
=
what
,
f
=
f
,
figure_key
=
figure_key
,
fig
=
fig
,
selection
=
selection
,
grid_before
=
grid_before
,
grid_limits
=
grid_limits
,
normalize
=
normalize
,
colormap
=
colormap
,
what_kwargs
=
what_kwargs
,
*
*
kwargs
)
if
show
:
plot2d
.
show
(
)
return
plot2d


def healpix_count(self, expression=None, healpix_expression=None, healpix_max_level=12, healpix_level=8, binby=None, limits=None, shape=default_shape, delay=False, progress=None, selection=None):
        """Count non missing value for expression on an array which represents healpix data.

        :param expression: Expression or column for which to count non-missing values, or None or '*' for counting the rows
        :param healpix_expression: {healpix_max_level}
        :param healpix_max_level: {healpix_max_level}
        :param healpix_level: {healpix_level}
        :param binby: {binby}, these dimension follow the first healpix dimension.
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :return:
        """
        # if binby is None:
        import healpy as hp
        if healpix_expression is None:
            if self.ucds.get("source_id", None) == 'meta.id;meta.main':  # we now assume we have gaia data
                healpix_expression = "source_id/34359738368"

        if healpix_expression is None:
            raise ValueError("no healpix_expression given, and was unable to guess")

        reduce_level = healpix_max_level - healpix_level
        NSIDE = 2**healpix_level
        nmax = hp.nside2npix(NSIDE)
        scaling = 4**reduce_level
        expr = "%s/%s" % (healpix_expression, scaling)
        binby = [expr] + ([] if binby is None else _ensure_list(binby))
        shape = (nmax,) + _expand_shape(shape, len(binby) - 1)
        epsilon = 1. / scaling / 2
        limits = [[-epsilon, nmax - epsilon]] + ([] if limits is None else limits)
        return self.count(expression, binby=binby, limits=limits, shape=shape, delay=delay, progress=progress, selection=selection)

Count non missing value for expression on an array which represents healpix data .


def healpix_plot(self, healpix_expression="source_id/34359738368", healpix_max_level=12, healpix_level=8, what="count(*)", selection=None,
                     grid=None,
                     healpix_input="equatorial", healpix_output="galactic", f=None,
                     colormap="afmhot", grid_limits=None, image_size=800, nest=True,
                     figsize=None, interactive=False, title="", smooth=None, show=False, colorbar=True,
                     rotation=(0, 0, 0), **kwargs):
        """Viz data in 2d using a healpix column.

        :param healpix_expression: {healpix_max_level}
        :param healpix_max_level: {healpix_max_level}
        :param healpix_level: {healpix_level}
        :param what: {what}
        :param selection: {selection}
        :param grid: {grid}
        :param healpix_input: Specificy if the healpix index is in "equatorial", "galactic" or "ecliptic".
        :param healpix_output: Plot in "equatorial", "galactic" or "ecliptic".
        :param f: function to apply to the data
        :param colormap: matplotlib colormap
        :param grid_limits: Optional sequence [minvalue, maxvalue] that determine the min and max value that map to the colormap (values below and above these are clipped to the the min/max). (default is [min(f(grid)), max(f(grid)))
        :param image_size: size for the image that healpy uses for rendering
        :param nest: If the healpix data is in nested (True) or ring (False)
        :param figsize: If given, modify the matplotlib figure size. Example (14,9)
        :param interactive: (Experimental, uses healpy.mollzoom is True)
        :param title: Title of figure
        :param smooth: apply gaussian smoothing, in degrees
        :param show: Call matplotlib's show (True) or not (False, defaut)
        :param rotation: Rotatate the plot, in format (lon, lat, psi) such that (lon, lat) is the center, and rotate on the screen by angle psi. All angles are degrees.
        :return:
        """
        # plot_level = healpix_level #healpix_max_level-reduce_level
        import healpy as hp
        import pylab as plt
        if grid is None:
            reduce_level = healpix_max_level - healpix_level
            NSIDE = 2**healpix_level
            nmax = hp.nside2npix(NSIDE)
            # print nmax, np.sqrt(nmax)
            scaling = 4**reduce_level
            # print nmax
            epsilon = 1. / scaling / 2
            grid = self._stat(what=what, binby="%s/%s" % (healpix_expression, scaling), limits=[-epsilon, nmax - epsilon], shape=nmax, selection=selection)
        if grid_limits:
            grid_min, grid_max = grid_limits
        else:
            grid_min = grid_max = None
        f_org = f
        f = _parse_f(f)
        if smooth:
            if nest:
                grid = hp.reorder(grid, inp="NEST", out="RING")
                nest = False
            # grid[np.isnan(grid)] = np.nanmean(grid)
            grid = hp.smoothing(grid, sigma=np.radians(smooth))
        fgrid = f(grid)
        coord_map = dict(equatorial='C', galactic='G', ecliptic="E")
        fig = plt.gcf()
        if figsize is not None:
            fig.set_size_inches(*figsize)
        what_label = what
        if f_org:
            what_label = f_org + " " + what_label
        f = hp.mollzoom if interactive else hp.mollview
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            coord = coord_map[healpix_input], coord_map[healpix_output]
            if coord_map[healpix_input] == coord_map[healpix_output]:
                coord = None
            f(fgrid, unit=what_label, rot=rotation, nest=nest, title=title, coord=coord,
              cmap=colormap, hold=True, xsize=image_size, min=grid_min, max=grid_max, cbar=colorbar, **kwargs)
        if show:
            plt.show()


def plot3d(self, x, y, z, vx=None, vy=None, vz=None, vwhat=None, limits=None, grid=None, what="count(*)", shape=128, selection=[None, True], f=None,
               vcount_limits=None,
               smooth_pre=None, smooth_post=None, grid_limits=None, normalize="normalize", colormap="afmhot",
               figure_key=None, fig=None,
               lighting=True, level=[0.1, 0.5, 0.9], opacity=[0.01, 0.05, 0.1], level_width=0.1,
               show=True, **kwargs):
        """Use at own risk, requires ipyvolume"""
        import vaex.ext.ipyvolume
        # vaex.ext.ipyvolume.
        cls = vaex.ext.ipyvolume.PlotDefault
        plot3d = cls(df=self, x=x, y=y, z=z, vx=vx, vy=vy, vz=vz,
                     grid=grid, shape=shape, limits=limits, what=what,
                     f=f, figure_key=figure_key, fig=fig,
                     selection=selection, smooth_pre=smooth_pre, smooth_post=smooth_post,
                     grid_limits=grid_limits, vcount_limits=vcount_limits, normalize=normalize, colormap=colormap, **kwargs)
        if show:
            plot3d.show()
        return plot3d

def
plot3d
(
self
,
x
,
y
,
z
,
vx
=
,
vy
=
,
vz
=
,
vwhat
=
,
limits
=
,
grid
=
,
what
=
"count(*)"
,
shape
=
128
,
selection
=
[
,
True
]
,
f
=
,
vcount_limits
=
,
smooth_pre
=
,
smooth_post
=
,
grid_limits
=
,
normalize
=
"normalize"
,
colormap
=
"afmhot"
,
figure_key
=
,
fig
=
,
lighting
=
True
,
level
=
[
0.1
,
0.5
,
0.9
]
,
opacity
=
[
0.01
,
0.05
,
0.1
]
,
level_width
=
0.1
,
show
=
True
,
*
*
kwargs
)
:
import
vaex
.
ext
.
ipyvolume
# vaex.ext.ipyvolume.
cls
=
vaex
.
ext
.
ipyvolume
.
PlotDefault
plot3d
=
cls
(
df
=
self
,
x
=
x
,
y
=
y
,
z
=
z
,
vx
=
vx
,
vy
=
vy
,
vz
=
vz
,
grid
=
grid
,
shape
=
shape
,
limits
=
limits
,
what
=
what
,
f
=
f
,
figure_key
=
figure_key
,
fig
=
fig
,
selection
=
selection
,
smooth_pre
=
smooth_pre
,
smooth_post
=
smooth_post
,
grid_limits
=
grid_limits
,
vcount_limits
=
vcount_limits
,
normalize
=
normalize
,
colormap
=
colormap
,
*
*
kwargs
)
if
show
:
plot3d
.
show
(
)
return
plot3d


def dtype(self, expression, internal=False):
        """Return the numpy dtype for the given expression, if not a column, the first row will be evaluated to get the dtype."""
        expression = _ensure_string_from_expression(expression)
        if expression in self.variables:
            return np.float64(1).dtype
        elif expression in self.columns.keys():
            column = self.columns[expression]
            data = column[0:1]
            dtype = data.dtype
        else:
            data = self.evaluate(expression, 0, 1, filtered=False)
            dtype = data.dtype
        if not internal:
            if dtype != str_type:
                if dtype.kind in 'US':
                    return str_type
                if dtype.kind == 'O':
                    # we lie about arrays containing strings
                    if isinstance(data[0], six.string_types):
                        return str_type
        return dtype

Return the numpy dtype for the given expression if not a column the first row will be evaluated to get the dtype .


def get_private_dir(self, create=False):
        """Each DataFrame has a directory where files are stored for metadata etc.

        Example

        >>> import vaex
        >>> ds = vaex.example()
        >>> vaex.get_private_dir()
        '/Users/users/breddels/.vaex/dfs/_Users_users_breddels_vaex-testing_data_helmi-dezeeuw-2000-10p.hdf5'

        :param bool create: is True, it will create the directory if it does not exist
        """
        if self.is_local():
            name = os.path.abspath(self.path).replace(os.path.sep, "_")[:250]  # should not be too long for most os'es
            name = name.replace(":", "_")  # for windows drive names
        else:
            server = self.server
            name = "%s_%s_%s_%s" % (server.hostname, server.port, server.base_path.replace("/", "_"), self.name)
        dir = os.path.join(vaex.utils.get_private_dir(), "dfs", name)
        if create and not os.path.exists(dir):
            os.makedirs(dir)
        return dir

def
get_private_dir
(
self
,
create
=
False
)
:
if
self
.
is_local
(
)
:
name
=
os
.
path
.
abspath
(
self
.
path
)
.
replace
(
os
.
path
.
sep
,
"_"
)
[
:
250
]
# should not be too long for most os'es
name
=
name
.
replace
(
":"
,
"_"
)
# for windows drive names
else
:
server
=
self
.
server
name
=
"%s_%s_%s_%s"
%
(
server
.
hostname
,
server
.
port
,
server
.
base_path
.
replace
(
"/"
,
"_"
)
,
self
.
name
)
dir
=
os
.
path
.
join
(
vaex
.
utils
.
get_private_dir
(
)
,
"dfs"
,
name
)
if
create
and
not
os
.
path
.
exists
(
dir
)
:
os
.
makedirs
(
dir
)
return
dir

Each DataFrame has a directory where files are stored for metadata etc .


def state_get(self):
        """Return the internal state of the DataFrame in a dictionary

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, y=2)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5
        >>> df.state_get()
        {'active_range': [0, 1],
        'column_names': ['x', 'y', 'r'],
        'description': None,
        'descriptions': {},
        'functions': {},
        'renamed_columns': [],
        'selections': {'__filter__': None},
        'ucds': {},
        'units': {},
        'variables': {},
        'virtual_columns': {'r': '(((x ** 2) + (y ** 2)) ** 0.5)'}}
        """

        virtual_names = list(self.virtual_columns.keys()) + list(self.variables.keys())
        units = {key: str(value) for key, value in self.units.items()}
        ucds = {key: value for key, value in self.ucds.items() if key in virtual_names}
        descriptions = {key: value for key, value in self.descriptions.items()}
        import vaex.serialize

        def check(key, value):
            if not vaex.serialize.can_serialize(value.f):
                warnings.warn('Cannot serialize function for virtual column {} (use vaex.serialize.register)'.format(key))
                return False
            return True

        def clean(value):
            return vaex.serialize.to_dict(value.f)
        functions = {key: clean(value) for key, value in self.functions.items() if check(key, value)}
        virtual_columns = {key: value for key, value in self.virtual_columns.items()}
        selections = {name: self.get_selection(name) for name, history in self.selection_histories.items()}
        selections = {name: selection.to_dict() if selection is not None else None for name, selection in selections.items()}
    # if selection is not None}
        state = dict(virtual_columns=virtual_columns,
                     column_names=self.column_names,
                     renamed_columns=self._renamed_columns,
                     variables=self.variables,
                     functions=functions,
                     selections=selections,
                     ucds=ucds,
                     units=units,
                     descriptions=descriptions,
                     description=self.description,
                     active_range=[self._index_start, self._index_end])
        return state


def state_set(self, state, use_active_range=False):
        """Sets the internal state of the df

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, y=2)
        >>> df
          #    x    y        r
          0    1    2  2.23607
        >>> df['r'] = (df.x**2 + df.y**2)**0.5
        >>> state = df.state_get()
        >>> state
        {'active_range': [0, 1],
        'column_names': ['x', 'y', 'r'],
        'description': None,
        'descriptions': {},
        'functions': {},
        'renamed_columns': [],
        'selections': {'__filter__': None},
        'ucds': {},
        'units': {},
        'variables': {},
        'virtual_columns': {'r': '(((x ** 2) + (y ** 2)) ** 0.5)'}}
        >>> df2 = vaex.from_scalars(x=3, y=4)
        >>> df2.state_set(state)  # now the virtual functions are 'copied'
        >>> df2
          #    x    y    r
          0    3    4    5

        :param state: dict as returned by :meth:`DataFrame.state_get`.
        :param bool use_active_range: Whether to use the active range or not.
        """
        self.description = state['description']
        if use_active_range:
            self._index_start, self._index_end = state['active_range']
        self._length_unfiltered = self._index_end - self._index_start
        if 'renamed_columns' in state:
            for old, new in state['renamed_columns']:
                self._rename(old, new)
        for name, value in state['functions'].items():
            self.add_function(name, vaex.serialize.from_dict(value))
        if 'column_names' in state:
            # we clear all columns, and add them later on, since otherwise self[name] = ... will try
            # to rename the columns (which is unsupported for remote dfs)
            self.column_names = []
            self.virtual_columns = collections.OrderedDict()
            for name, value in state['virtual_columns'].items():
                self[name] = self._expr(value)
                # self._save_assign_expression(name)
            self.column_names = state['column_names']
        else:
            # old behaviour
            self.virtual_columns = collections.OrderedDict()
            for name, value in state['virtual_columns'].items():
                self[name] = self._expr(value)
        self.variables = state['variables']
        import astropy  # TODO: make this dep optional?
        units = {key: astropy.units.Unit(value) for key, value in state["units"].items()}
        self.units.update(units)
        for name, selection_dict in state['selections'].items():
            # TODO: make selection use the vaex.serialize framework
            if selection_dict is None:
                selection = None
            else:
                selection = selections.selection_from_dict(selection_dict)
            self.set_selection(selection, name=name)


def remove_virtual_meta(self):
        """Removes the file with the virtual column etc, it does not change the current virtual columns etc."""
        dir = self.get_private_dir(create=True)
        path = os.path.join(dir, "virtual_meta.yaml")
        try:
            if os.path.exists(path):
                os.remove(path)
            if not os.listdir(dir):
                os.rmdir(dir)
        except:
            logger.exception("error while trying to remove %s or %s", path, dir)

def
remove_virtual_meta
(
self
)
:
dir
=
self
.
get_private_dir
(
create
=
True
)
path
=
os
.
path
.
join
(
dir
,
"virtual_meta.yaml"
)
try
:
if
os
.
path
.
exists
(
path
)
:
os
.
remove
(
path
)
if
not
os
.
listdir
(
dir
)
:
os
.
rmdir
(
dir
)
except
:
logger
.
exception
(
"error while trying to remove %s or %s"
,
path
,
dir
)

Removes the file with the virtual column etc it does not change the current virtual columns etc .


def write_virtual_meta(self):
        """Writes virtual columns, variables and their ucd,description and units.

        The default implementation is to write this to a file called virtual_meta.yaml in the directory defined by
        :func:`DataFrame.get_private_dir`. Other implementation may store this in the DataFrame file itself.

        This method is called after virtual columns or variables are added. Upon opening a file, :func:`DataFrame.update_virtual_meta`
        is called, so that the information is not lost between sessions.

        Note: opening a DataFrame twice may result in corruption of this file.

        """
        path = os.path.join(self.get_private_dir(create=True), "virtual_meta.yaml")
        virtual_names = list(self.virtual_columns.keys()) + list(self.variables.keys())
        units = {key: str(value) for key, value in self.units.items() if key in virtual_names}
        ucds = {key: value for key, value in self.ucds.items() if key in virtual_names}
        descriptions = {key: value for key, value in self.descriptions.items() if key in virtual_names}
        meta_info = dict(virtual_columns=self.virtual_columns,
                         variables=self.variables,
                         ucds=ucds, units=units, descriptions=descriptions)
        vaex.utils.write_json_or_yaml(path, meta_info)

def write_virtual_meta ( self ) : path = os . path . join ( self . get_private_dir ( create = True ) , "virtual_meta.yaml" ) virtual_names = list ( self . virtual_columns . keys ( ) ) + list ( self . variables . keys ( ) ) units = { key : str ( value ) for key , value in self . units . items ( ) if key in virtual_names } ucds = { key : value for key , value in self . ucds . items ( ) if key in virtual_names } descriptions = { key : value for key , value in self . descriptions . items ( ) if key in virtual_names } meta_info = dict ( virtual_columns = self . virtual_columns , variables = self . variables , ucds = ucds , units = units , descriptions = descriptions ) vaex . utils . write_json_or_yaml ( path , meta_info )

Writes virtual columns variables and their ucd description and units .


def write_meta(self):
        """Writes all meta data, ucd,description and units

        The default implementation is to write this to a file called meta.yaml in the directory defined by
        :func:`DataFrame.get_private_dir`. Other implementation may store this in the DataFrame file itself.
        (For instance the vaex hdf5 implementation does this)

        This method is called after virtual columns or variables are added. Upon opening a file, :func:`DataFrame.update_meta`
        is called, so that the information is not lost between sessions.

        Note: opening a DataFrame twice may result in corruption of this file.

        """
        # raise NotImplementedError
        path = os.path.join(self.get_private_dir(create=True), "meta.yaml")
        units = {key: str(value) for key, value in self.units.items()}
        meta_info = dict(description=self.description,
                         ucds=self.ucds, units=units, descriptions=self.descriptions,
                         )
        vaex.utils.write_json_or_yaml(path, meta_info)

def
write_meta
(
self
)
:
# raise NotImplementedError
path
=
os
.
path
.
join
(
self
.
get_private_dir
(
create
=
True
)
,
"meta.yaml"
)
units
=
{
key
:
str
(
value
)
for
key
,
value
in
self
.
units
.
items
(
)
}
meta_info
=
dict
(
description
=
self
.
description
,
ucds
=
self
.
ucds
,
units
=
units
,
descriptions
=
self
.
descriptions
,
)
vaex
.
utils
.
write_json_or_yaml
(
path
,
meta_info
)

Writes all meta data ucd description and units


def subspaces(self, expressions_list=None, dimensions=None, exclude=None, **kwargs):
        """Generate a Subspaces object, based on a custom list of expressions or all possible combinations based on
        dimension

        :param expressions_list: list of list of expressions, where the inner list defines the subspace
        :param dimensions: if given, generates a subspace with all possible combinations for that dimension
        :param exclude: list of
        """
        if dimensions is not None:
            expressions_list = list(itertools.combinations(self.get_column_names(), dimensions))
            if exclude is not None:
                import six

                def excluded(expressions):
                    if callable(exclude):
                        return exclude(expressions)
                    elif isinstance(exclude, six.string_types):
                        return exclude in expressions
                    elif isinstance(exclude, (list, tuple)):
                        # $#expressions = set(expressions)
                        for e in exclude:
                            if isinstance(e, six.string_types):
                                if e in expressions:
                                    return True
                            elif isinstance(e, (list, tuple)):
                                if set(e).issubset(expressions):
                                    return True
                            else:
                                raise ValueError("elements of exclude should contain a string or a sequence of strings")
                    else:
                        raise ValueError("exclude should contain a string, a sequence of strings, or should be a callable")
                    return False
                # test if any of the elements of exclude are a subset of the expression
                expressions_list = [expr for expr in expressions_list if not excluded(expr)]
            logger.debug("expression list generated: %r", expressions_list)
        import vaex.legacy
        return vaex.legacy.Subspaces([self(*expressions, **kwargs) for expressions in expressions_list])

Generate a Subspaces object based on a custom list of expressions or all possible combinations based on dimension


def set_variable(self, name, expression_or_value, write=True):
        """Set the variable to an expression or value defined by expression_or_value.

        Example

        >>> df.set_variable("a", 2.)
        >>> df.set_variable("b", "a**2")
        >>> df.get_variable("b")
        'a**2'
        >>> df.evaluate_variable("b")
        4.0

        :param name: Name of the variable
        :param write: write variable to meta file
        :param expression: value or expression
        """
        self.variables[name] = expression_or_value

def set_variable ( self , name , expression_or_value , write = True ) : self . variables [ name ] = expression_or_value

Set the variable to an expression or value defined by expression_or_value .


def evaluate_variable(self, name):
        """Evaluates the variable given by name."""
        if isinstance(self.variables[name], six.string_types):
            # TODO: this does not allow more than one level deep variable, like a depends on b, b on c, c is a const
            value = eval(self.variables[name], expression_namespace, self.variables)
            return value
        else:
            return self.variables[name]


def _evaluate_selection_mask(self, name="default", i1=None, i2=None, selection=None, cache=False):
        """Internal use, ignores the filter"""
        i1 = i1 or 0
        i2 = i2 or len(self)
        scope = scopes._BlockScopeSelection(self, i1, i2, selection, cache=cache)
        return scope.evaluate(name)

def _evaluate_selection_mask ( self , name = "default" , i1 = None , i2 = None , selection = None , cache = False ) : i1 = i1 or 0 i2 = i2 or len ( self ) scope = scopes . _BlockScopeSelection ( self , i1 , i2 , selection , cache = cache ) return scope . evaluate ( name )


def to_dict(self, column_names=None, selection=None, strings=True, virtual=False):
        """Return a dict containing the ndarray corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :return: dict
        """
        return dict(self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=virtual))

def to_dict ( self , column_names = None , selection = None , strings = True , virtual = False ) : return dict ( self . to_items ( column_names = column_names , selection = selection , strings = strings , virtual = virtual ) )


def to_copy(self, column_names=None, selection=None, strings=True, virtual=False, selections=True):
        """Return a copy of the DataFrame, if selection is None, it does not copy the data, it just has a reference

        :param column_names: list of column names, to copy, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param selections: copy selections to a new DataFrame
        :return: dict
        """
        if column_names:
            column_names = _ensure_strings_from_expressions(column_names)
        df = vaex.from_items(*self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=False))
        if virtual:
            for name, value in self.virtual_columns.items():
                df.add_virtual_column(name, value)
        if selections:
            # the filter selection does not need copying
            for key, value in self.selection_histories.items():
                if key != FILTER_SELECTION_NAME:
                    df.selection_histories[key] = list(value)
            for key, value in self.selection_history_indices.items():
                if key != FILTER_SELECTION_NAME:
                    df.selection_history_indices[key] = value
        df.functions.update(self.functions)
        df.copy_metadata(self)
        return df

Return a copy of the DataFrame if selection is None it does not copy the data it just has a reference


def to_pandas_df(self, column_names=None, selection=None, strings=True, virtual=False, index_name=None):
        """Return a pandas DataFrame containing the ndarray corresponding to the evaluated data

         If index is given, that column is used for the index of the dataframe.

         Example

         >>> df_pandas = df.to_pandas_df(["x", "y", "z"])
         >>> df_copy = vaex.from_pandas(df_pandas)

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param index_column: if this column is given it is used for the index of the DataFrame
        :return: pandas.DataFrame object
        """
        import pandas as pd
        data = self.to_dict(column_names=column_names, selection=selection, strings=strings, virtual=virtual)
        if index_name is not None:
            if index_name in data:
                index = data.pop(index_name)
            else:
                index = self.evaluate(index_name, selection=selection)
        else:
            index = None
        df = pd.DataFrame(data=data, index=index)
        if index is not None:
            df.index.name = index_name
        return df

def to_pandas_df ( self , column_names = None , selection = None , strings = True , virtual = False , index_name = None ) : import pandas as pd data = self . to_dict ( column_names = column_names , selection = selection , strings = strings , virtual = virtual ) if index_name is not None : if index_name in data : index = data . pop ( index_name ) else : index = self . evaluate ( index_name , selection = selection ) else : index = None df = pd . DataFrame ( data = data , index = index ) if index is not None : df . index . name = index_name return df


def to_arrow_table(self, column_names=None, selection=None, strings=True, virtual=False):
        """Returns an arrow Table object containing the arrays corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :return: pyarrow.Table object
        """
        from vaex_arrow.convert import arrow_table_from_vaex_df
        return arrow_table_from_vaex_df(self, column_names, selection, strings, virtual)

def to_arrow_table ( self , column_names = None , selection = None , strings = True , virtual = False ) : from vaex_arrow . convert import arrow_table_from_vaex_df return arrow_table_from_vaex_df ( self , column_names , selection , strings , virtual )


def to_astropy_table(self, column_names=None, selection=None, strings=True, virtual=False, index=None):
        """Returns a astropy table object containing the ndarrays corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param index: if this column is given it is used for the index of the DataFrame
        :return: astropy.table.Table object
        """
        from astropy.table import Table, Column, MaskedColumn
        meta = dict()
        meta["name"] = self.name
        meta["description"] = self.description

        table = Table(meta=meta)
        for name, data in self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=virtual):
            if self.dtype(name) == str_type:  # for astropy we convert it to unicode, it seems to ignore object type
                data = np.array(data).astype('U')
            meta = dict()
            if name in self.ucds:
                meta["ucd"] = self.ucds[name]
            if np.ma.isMaskedArray(data):
                cls = MaskedColumn
            else:
                cls = Column
            table[name] = cls(data, unit=self.unit(name), description=self.descriptions.get(name), meta=meta)
        return table


def add_column(self, name, f_or_array):
        """Add an in memory array as a column."""
        if isinstance(f_or_array, (np.ndarray, Column)):
            data = ar = f_or_array
            # it can be None when we have an 'empty' DataFrameArrays
            if self._length_original is None:
                self._length_unfiltered = _len(data)
                self._length_original = _len(data)
                self._index_end = self._length_unfiltered
            if _len(ar) != self.length_original():
                if self.filtered:
                    # give a better warning to avoid confusion
                    if len(self) == len(ar):
                        raise ValueError("Array is of length %s, while the length of the DataFrame is %s due to the filtering, the (unfiltered) length is %s." % (len(ar), len(self), self.length_unfiltered()))
                raise ValueError("array is of length %s, while the length of the DataFrame is %s" % (len(ar), self.length_original()))
            # assert self.length_unfiltered() == len(data), "columns should be of equal length, length should be %d, while it is %d" % ( self.length_unfiltered(), len(data))
            self.columns[name] = f_or_array
            if name not in self.column_names:
                self.column_names.append(name)
        else:
            raise ValueError("functions not yet implemented")
        self._save_assign_expression(name, Expression(self, name))


def rename_column(self, name, new_name, unique=False, store_in_state=True):
        """Renames a column, not this is only the in memory name, this will not be reflected on disk"""
        new_name = vaex.utils.find_valid_name(new_name, used=[] if not unique else list(self))
        data = self.columns.get(name)
        if data is not None:
            del self.columns[name]
            self.column_names[self.column_names.index(name)] = new_name
            self.columns[new_name] = data
        else:
            expression = self.virtual_columns[name]
            del self.virtual_columns[name]
            self.virtual_columns[new_name] = expression
        if store_in_state:
            self._renamed_columns.append((name, new_name))
        for d in [self.ucds, self.units, self.descriptions]:
            if name in d:
                d[new_name] = d[name]
                del d[name]
        return new_name

def rename_column ( self , name , new_name , unique = False , store_in_state = True ) : new_name = vaex . utils . find_valid_name ( new_name , used = [ ] if not unique else list ( self ) ) data = self . columns . get ( name ) if data is not None : del self . columns [ name ] self . column_names [ self . column_names . index ( name ) ] = new_name self . columns [ new_name ] = data else : expression = self . virtual_columns [ name ] del self . virtual_columns [ name ] self . virtual_columns [ new_name ] = expression if store_in_state : self . _renamed_columns . append ( ( name , new_name ) ) for d in [ self . ucds , self . units , self . descriptions ] : if name in d : d [ new_name ] = d [ name ] del d [ name ] return new_name

Renames a column not this is only the in memory name this will not be reflected on disk


def add_virtual_columns_cartesian_to_polar(self, x="x", y="y", radius_out="r_polar", azimuth_out="phi_polar",
                                               propagate_uncertainties=False,
                                               radians=False):
        """Convert cartesian to polar coordinates

        :param x: expression for x
        :param y: expression for y
        :param radius_out: name for the virtual column for the radius
        :param azimuth_out: name for the virtual column for the azimuth angle
        :param propagate_uncertainties: {propagate_uncertainties}
        :param radians: if True, azimuth is in radians, defaults to degrees
        :return:
        """
        x = self[x]
        y = self[y]
        if radians:
            to_degrees = ""
        else:
            to_degrees = "*180/pi"
        r = np.sqrt(x**2 + y**2)
        self[radius_out] = r
        phi = np.arctan2(y, x)
        if not radians:
            phi = phi * 180/np.pi
        self[azimuth_out] = phi
        if propagate_uncertainties:
            self.propagate_uncertainties([self[radius_out], self[azimuth_out]])

def add_virtual_columns_cartesian_to_polar ( self , x = "x" , y = "y" , radius_out = "r_polar" , azimuth_out = "phi_polar" , propagate_uncertainties = False , radians = False ) : x = self [ x ] y = self [ y ] if radians : to_degrees = "" else : to_degrees = "*180/pi" r = np . sqrt ( x ** 2 + y ** 2 ) self [ radius_out ] = r phi = np . arctan2 ( y , x ) if not radians : phi = phi * 180 / np . pi self [ azimuth_out ] = phi if propagate_uncertainties : self . propagate_uncertainties ( [ self [ radius_out ] , self [ azimuth_out ] ] )


def add_virtual_columns_cartesian_velocities_to_spherical(self, x="x", y="y", z="z", vx="vx", vy="vy", vz="vz", vr="vr", vlong="vlong", vlat="vlat", distance=None):
        """Concert velocities from a cartesian to a spherical coordinate system

        TODO: errors

        :param x: name of x column (input)
        :param y:         y
        :param z:         z
        :param vx:       vx
        :param vy:       vy
        :param vz:       vz
        :param vr: name of the column for the radial velocity in the r direction (output)
        :param vlong: name of the column for the velocity component in the longitude direction  (output)
        :param vlat: name of the column for the velocity component in the latitude direction, positive points to the north pole (output)
        :param distance: Expression for distance, if not given defaults to sqrt(x**2+y**2+z**2), but if this column already exists, passing this expression may lead to a better performance
        :return:
        """
        # see http://www.astrosurf.com/jephem/library/li110spherCart_en.htm
        if distance is None:
            distance = "sqrt({x}**2+{y}**2+{z}**2)".format(**locals())
        self.add_virtual_column(vr, "({x}*{vx}+{y}*{vy}+{z}*{vz})/{distance}".format(**locals()))
        self.add_virtual_column(vlong, "-({vx}*{y}-{x}*{vy})/sqrt({x}**2+{y}**2)".format(**locals()))
        self.add_virtual_column(vlat, "-({z}*({x}*{vx}+{y}*{vy}) - ({x}**2+{y}**2)*{vz})/( {distance}*sqrt({x}**2+{y}**2) )".format(**locals()))


def add_virtual_columns_cartesian_velocities_to_polar(self, x="x", y="y", vx="vx", radius_polar=None, vy="vy", vr_out="vr_polar", vazimuth_out="vphi_polar",
                                                          propagate_uncertainties=False,):
        """Convert cartesian to polar velocities.

        :param x:
        :param y:
        :param vx:
        :param radius_polar: Optional expression for the radius, may lead to a better performance when given.
        :param vy:
        :param vr_out:
        :param vazimuth_out:
        :param propagate_uncertainties: {propagate_uncertainties}
        :return:
        """
        x = self._expr(x)
        y = self._expr(y)
        vx = self._expr(vx)
        vy = self._expr(vy)
        if radius_polar is None:
            radius_polar = np.sqrt(x**2 + y**2)
        radius_polar = self._expr(radius_polar)
        self[vr_out]       = (x*vx + y*vy) / radius_polar
        self[vazimuth_out] = (x*vy - y*vx) / radius_polar
        if propagate_uncertainties:
            self.propagate_uncertainties([self[vr_out], self[vazimuth_out]])


def add_virtual_columns_polar_velocities_to_cartesian(self, x='x', y='y', azimuth=None, vr='vr_polar', vazimuth='vphi_polar', vx_out='vx', vy_out='vy', propagate_uncertainties=False):
        """ Convert cylindrical polar velocities to Cartesian.

        :param x:
        :param y:
        :param azimuth: Optional expression for the azimuth in degrees , may lead to a better performance when given.
        :param vr:
        :param vazimuth:
        :param vx_out:
        :param vy_out:
        :param propagate_uncertainties: {propagate_uncertainties}
        """
        x = self._expr(x)
        y = self._expr(y)
        vr = self._expr(vr)
        vazimuth = self._expr(vazimuth)
        if azimuth is not None:
            azimuth = self._expr(azimuth)
            azimuth = np.deg2rad(azimuth)
        else:
            azimuth = np.arctan2(y, x)
        azimuth = self._expr(azimuth)
        self[vx_out] = vr * np.cos(azimuth) - vazimuth * np.sin(azimuth)
        self[vy_out] = vr * np.sin(azimuth) + vazimuth * np.cos(azimuth)
        if propagate_uncertainties:
            self.propagate_uncertainties([self[vx_out], self[vy_out]])

Convert cylindrical polar velocities to Cartesian .


def add_virtual_columns_rotation(self, x, y, xnew, ynew, angle_degrees, propagate_uncertainties=False):
        """Rotation in 2d.

        :param str x: Name/expression of x column
        :param str y: idem for y
        :param str xnew: name of transformed x column
        :param str ynew:
        :param float angle_degrees: rotation in degrees, anti clockwise
        :return:
        """
        x = _ensure_string_from_expression(x)
        y = _ensure_string_from_expression(y)
        theta = np.radians(angle_degrees)
        matrix = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
        m = matrix_name = x + "_" + y + "_rot"
        for i in range(2):
            for j in range(2):
                self.set_variable(matrix_name + "_%d%d" % (i, j), matrix[i, j].item())
        self[xnew] = self._expr("{m}_00 * {x} + {m}_01 * {y}".format(**locals()))
        self[ynew] = self._expr("{m}_10 * {x} + {m}_11 * {y}".format(**locals()))
        if propagate_uncertainties:
            self.propagate_uncertainties([self[xnew], self[ynew]])

def
add_virtual_columns_rotation
(
self
,
x
,
y
,
xnew
,
ynew
,
angle_degrees
,
propagate_uncertainties
=
False
)
:
x
=
_ensure_string_from_expression
(
x
)
y
=
_ensure_string_from_expression
(
y
)
theta
=
np
.
radians
(
angle_degrees
)
matrix
=
np
.
array
(
[
[
np
.
cos
(
theta
)
,
np
.
sin
(
theta
)
]
,
[
np
.
sin
(
theta
)
,
np
.
cos
(
theta
)
]
]
)
m
=
matrix_name
=
x
+
"_"
+
y
+
"_rot"
for
i
in
range
(
2
)
:
for
j
in
range
(
2
)
:
self
.
set_variable
(
matrix_name
+
"_%d%d"
%
(
i
,
j
)
,
matrix
[
i
,
j
]
.
item
(
)
)
self
[
xnew
]
=
self
.
_expr
(
"{m}_00 * {x} + {m}_01 * {y}"
.
format
(
*
*
locals
(
)
)
)
self
[
ynew
]
=
self
.
_expr
(
"{m}_10 * {x} + {m}_11 * {y}"
.
format
(
*
*
locals
(
)
)
)
if
propagate_uncertainties
:
self
.
propagate_uncertainties
(
[
self
[
xnew
]
,
self
[
ynew
]
]
)


def add_virtual_columns_spherical_to_cartesian(self, alpha, delta, distance, xname="x", yname="y", zname="z",
                                                   propagate_uncertainties=False,
                                                   center=[0, 0, 0], center_name="solar_position", radians=False):
        """Convert spherical to cartesian coordinates.

        :param alpha:
        :param delta: polar angle, ranging from the -90 (south pole) to 90 (north pole)
        :param distance: radial distance, determines the units of x, y and z
        :param xname:
        :param yname:
        :param zname:
        :param propagate_uncertainties: {propagate_uncertainties}
        :param center:
        :param center_name:
        :param radians:
        :return:
        """
        alpha = self._expr(alpha)
        delta = self._expr(delta)
        distance = self._expr(distance)
        if not radians:
            alpha = alpha * self._expr('pi')/180
            delta = delta * self._expr('pi')/180

        # TODO: use sth like .optimize by default to get rid of the +0 ?
        if center[0]:
            self[xname] = np.cos(alpha) * np.cos(delta) * distance + center[0]
        else:
            self[xname] = np.cos(alpha) * np.cos(delta) * distance
        if center[1]:
            self[yname] = np.sin(alpha) * np.cos(delta) * distance + center[1]
        else:
            self[yname] = np.sin(alpha) * np.cos(delta) * distance
        if center[2]:
            self[zname] =                 np.sin(delta) * distance + center[2]
        else:
            self[zname] =                 np.sin(delta) * distance
        if propagate_uncertainties:
            self.propagate_uncertainties([self[xname], self[yname], self[zname]])

Convert spherical to cartesian coordinates .


def add_virtual_columns_cartesian_to_spherical(self, x="x", y="y", z="z", alpha="l", delta="b", distance="distance", radians=False, center=None, center_name="solar_position"):
        """Convert cartesian to spherical coordinates.

        :param x:
        :param y:
        :param z:
        :param alpha:
        :param delta: name for polar angle, ranges from -90 to 90 (or -pi to pi when radians is True).
        :param distance:
        :param radians:
        :param center:
        :param center_name:
        :return:
        """
        transform = "" if radians else "*180./pi"

        if center is not None:
            self.add_variable(center_name, center)
        if center is not None and center[0] != 0:
            x = "({x} - {center_name}[0])".format(**locals())
        if center is not None and center[1] != 0:
            y = "({y} - {center_name}[1])".format(**locals())
        if center is not None and center[2] != 0:
            z = "({z} - {center_name}[2])".format(**locals())
        self.add_virtual_column(distance, "sqrt({x}**2 + {y}**2 + {z}**2)".format(**locals()))
        # self.add_virtual_column(alpha, "((arctan2({y}, {x}) + 2*pi) % (2*pi)){transform}".format(**locals()))
        self.add_virtual_column(alpha, "arctan2({y}, {x}){transform}".format(**locals()))
        self.add_virtual_column(delta, "(-arccos({z}/{distance})+pi/2){transform}".format(**locals()))

Convert cartesian to spherical coordinates .


def add_virtual_column(self, name, expression, unique=False):
        """Add a virtual column to the DataFrame.

        Example:

        >>> df.add_virtual_column("r", "sqrt(x**2 + y**2 + z**2)")
        >>> df.select("r < 10")

        :param: str name: name of virtual column
        :param: expression: expression for the column
        :param str unique: if name is already used, make it unique by adding a postfix, e.g. _1, or _2
        """
        type = "change" if name in self.virtual_columns else "add"
        expression = _ensure_string_from_expression(expression)
        if name in self.get_column_names(virtual=False):
            renamed = '__' +vaex.utils.find_valid_name(name, used=self.get_column_names())
            expression = self._rename(name, renamed, expression)[0].expression

        name = vaex.utils.find_valid_name(name, used=[] if not unique else self.get_column_names())
        self.virtual_columns[name] = expression
        self.column_names.append(name)
        self._save_assign_expression(name)
        self.signal_column_changed.emit(self, name, "add")

def add_virtual_column ( self , name , expression , unique = False ) : type = "change" if name in self . virtual_columns else "add" expression = _ensure_string_from_expression ( expression ) if name in self . get_column_names ( virtual = False ) : renamed = '__' + vaex . utils . find_valid_name ( name , used = self . get_column_names ( ) ) expression = self . _rename ( name , renamed , expression ) [ 0 ] . expression name = vaex . utils . find_valid_name ( name , used = [ ] if not unique else self . get_column_names ( ) ) self . virtual_columns [ name ] = expression self . column_names . append ( name ) self . _save_assign_expression ( name ) self . signal_column_changed . emit ( self , name , "add" )


def delete_virtual_column(self, name):
        """Deletes a virtual column from a DataFrame."""
        del self.virtual_columns[name]
        self.signal_column_changed.emit(self, name, "delete")

def delete_virtual_column ( self , name ) : del self . virtual_columns [ name ] self . signal_column_changed . emit ( self , name , "delete" )

Deletes a virtual column from a DataFrame .


def add_variable(self, name, expression, overwrite=True, unique=True):
        """Add a variable to to a DataFrame.

        A variable may refer to other variables, and virtual columns and expression may refer to variables.

        Example

        >>> df.add_variable('center', 0)
        >>> df.add_virtual_column('x_prime', 'x-center')
        >>> df.select('x_prime < 0')

        :param: str name: name of virtual varible
        :param: expression: expression for the variable
        """
        if unique or overwrite or name not in self.variables:
            existing_names = self.get_column_names(virtual=False) + list(self.variables.keys())
            name = vaex.utils.find_valid_name(name, used=[] if not unique else existing_names)
            self.variables[name] = expression
            self.signal_variable_changed.emit(self, name, "add")
            if unique:
                return name

def add_variable ( self , name , expression , overwrite = True , unique = True ) : if unique or overwrite or name not in self . variables : existing_names = self . get_column_names ( virtual = False ) + list ( self . variables . keys ( ) ) name = vaex . utils . find_valid_name ( name , used = [ ] if not unique else existing_names ) self . variables [ name ] = expression self . signal_variable_changed . emit ( self , name , "add" ) if unique : return name


def delete_variable(self, name):
        """Deletes a variable from a DataFrame."""
        del self.variables[name]
        self.signal_variable_changed.emit(self, name, "delete")

def delete_variable ( self , name ) : del self . variables [ name ] self . signal_variable_changed . emit ( self , name , "delete" )


def tail(self, n=10):
        """Return a shallow copy a DataFrame with the last n rows."""
        N = len(self)
        # self.cat(i1=max(0, N-n), i2=min(len(self), N))
        return self[max(0, N - n):min(len(self), N)]

Return a shallow copy a DataFrame with the last n rows .


def head_and_tail_print(self, n=5):
        """Display the first and last n elements of a DataFrame."""
        from IPython import display
        display.display(display.HTML(self._head_and_tail_table(n)))

def head_and_tail_print ( self , n = 5 ) : from IPython import display display . display ( display . HTML ( self . _head_and_tail_table ( n ) ) )

Display the first and last n elements of a DataFrame .


def describe(self, strings=True, virtual=True, selection=None):
        """Give a description of the DataFrame.

        >>> import vaex
        >>> df = vaex.example()[['x', 'y', 'z']]
        >>> df.describe()
                         x          y          z
        dtype      float64    float64    float64
        count       330000     330000     330000
        missing          0          0          0
        mean    -0.0671315 -0.0535899  0.0169582
        std        7.31746    7.78605    5.05521
        min       -128.294   -71.5524   -44.3342
        max        271.366    146.466    50.7185
        >>> df.describe(selection=df.x > 0)
                           x         y          z
        dtype        float64   float64    float64
        count         164060    164060     164060
        missing       165940    165940     165940
        mean         5.13572 -0.486786 -0.0868073
        std          5.18701   7.61621    5.02831
        min      1.51635e-05  -71.5524   -44.3342
        max          271.366   78.0724    40.2191

        :param bool strings: Describe string columns or not
        :param bool virtual: Describe virtual columns or not
        :param selection: Optional selection to use.
        :return: Pandas dataframe

        """
        import pandas as pd
        N = len(self)
        columns = {}
        for feature in self.get_column_names(strings=strings, virtual=virtual)[:]:
            dtype = str(self.dtype(feature)) if self.dtype(feature) != str else 'str'
            if self.dtype(feature) == str_type or self.dtype(feature).kind in ['S', 'U', 'O']:
                count = self.count(feature, selection=selection, delay=True)
                self.execute()
                count = count.get()
                columns[feature] = ((dtype, count, N-count, '--', '--', '--', '--'))
            else:
                count = self.count(feature, selection=selection, delay=True)
                mean = self.mean(feature, selection=selection, delay=True)
                std = self.std(feature, selection=selection, delay=True)
                minmax = self.minmax(feature, selection=selection, delay=True)
                self.execute()
                count, mean, std, minmax = count.get(), mean.get(), std.get(), minmax.get()
                count = int(count)
                columns[feature] = ((dtype, count, N-count, mean, std, minmax[0], minmax[1]))
        return pd.DataFrame(data=columns, index=['dtype', 'count', 'missing', 'mean', 'std', 'min', 'max'])

def describe ( self , strings = True , virtual = True , selection = None ) : import pandas as pd N = len ( self ) columns = { } for feature in self . get_column_names ( strings = strings , virtual = virtual ) [ : ] : dtype = str ( self . dtype ( feature ) ) if self . dtype ( feature ) != str else 'str' if self . dtype ( feature ) == str_type or self . dtype ( feature ) . kind in [ 'S' , 'U' , 'O' ] : count = self . count ( feature , selection = selection , delay = True ) self . execute ( ) count = count . get ( ) columns [ feature ] = ( ( dtype , count , N - count , '--' , '--' , '--' , '--' ) ) else : count = self . count ( feature , selection = selection , delay = True ) mean = self . mean ( feature , selection = selection , delay = True ) std = self . std ( feature , selection = selection , delay = True ) minmax = self . minmax ( feature , selection = selection , delay = True ) self . execute ( ) count , mean , std , minmax = count . get ( ) , mean . get ( ) , std . get ( ) , minmax . get ( ) count = int ( count ) columns [ feature ] = ( ( dtype , count , N - count , mean , std , minmax [ 0 ] , minmax [ 1 ] ) ) return pd . DataFrame ( data = columns , index = [ 'dtype' , 'count' , 'missing' , 'mean' , 'std' , 'min' , 'max' ] )


def cat(self, i1, i2, format='html'):
        """Display the DataFrame from row i1 till i2

        For format, see https://pypi.org/project/tabulate/

        :param int i1: Start row
        :param int i2: End row.
        :param str format: Format to use, e.g. 'html', 'plain', 'latex'
        """
        from IPython import display
        if format == 'html':
            output = self._as_html_table(i1, i2)
            display.display(display.HTML(output))
        else:
            output = self._as_table(i1, i2, format=format)
            print(output)

def cat ( self , i1 , i2 , format = 'html' ) : from IPython import display if format == 'html' : output = self . _as_html_table ( i1 , i2 ) display . display ( display . HTML ( output ) ) else : output = self . _as_table ( i1 , i2 , format = format ) print ( output )


def set_current_row(self, value):
        """Set the current row, and emit the signal signal_pick."""
        if (value is not None) and ((value < 0) or (value >= len(self))):
            raise IndexError("index %d out of range [0,%d]" % (value, len(self)))
        self._current_row = value
        self.signal_pick.emit(self, value)

def
set_current_row
(
self
,
value
)
:
if
(
value
is
not
)
and
(
(
value
<
0
)
or
(
value
>=
len
(
self
)
)
)
:
raise
IndexError
(
"index %d out of range [0,%d]"
%
(
value
,
len
(
self
)
)
)
self
.
_current_row
=
value
self
.
signal_pick
.
emit
(
self
,
value
)

Set the current row and emit the signal signal_pick .


def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None):
        """Return a list of column names

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']

        :param virtual: If False, skip virtual columns
        :param hidden: If False, skip hidden columns
        :param strings: If False, skip string columns
        :param regex: Only return column names matching the (optional) regular expression
        :rtype: list of str

        Example:
        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']
        """
        def column_filter(name):
            '''Return True if column with specified name should be returned'''
            if regex and not re.match(regex, name):
                return False
            if not virtual and name in self.virtual_columns:
                return False
            if not strings and (self.dtype(name) == str_type or self.dtype(name).type == np.string_):
                return False
            if not hidden and name.startswith('__'):
                return False
            return True
        return [name for name in self.column_names if column_filter(name)]


def trim(self, inplace=False):
        '''Return a DataFrame, where all columns are 'trimmed' by the active range.

        For the returned DataFrame, df.get_active_range() returns (0, df.length_original()).

        {note_copy}

        :param inplace: {inplace}
        :rtype: DataFrame
        '''
        df = self if inplace else self.copy()
        for name in df:
            column = df.columns.get(name)
            if column is not None:
                if self._index_start == 0 and len(column) == self._index_end:
                    pass  # we already assigned it in .copy
                else:
                    if isinstance(column, np.ndarray):  # real array
                        df.columns[name] = column[self._index_start:self._index_end]
                    else:
                        df.columns[name] = column.trim(self._index_start, self._index_end)
        df._length_original = self.length_unfiltered()
        df._length_unfiltered = df._length_original
        df._index_start = 0
        df._index_end = df._length_original
        df._active_fraction = 1
        return df

def
trim
(
self
,
inplace
=
False
)
:
df
=
self
if
inplace
else
self
.
copy
(
)
for
name
in
df
:
column
=
df
.
columns
.
get
(
name
)
if
column
is
not
:
if
self
.
_index_start
==
0
and
len
(
column
)
==
self
.
_index_end
:
pass
# we already assigned it in .copy
else
:
if
isinstance
(
column
,
np
.
ndarray
)
:
# real array
df
.
columns
[
name
]
=
column
[
self
.
_index_start
:
self
.
_index_end
]
else
:
df
.
columns
[
name
]
=
column
.
trim
(
self
.
_index_start
,
self
.
_index_end
)
df
.
_length_original
=
self
.
length_unfiltered
(
)
df
.
_length_unfiltered
=
df
.
_length_original
df
.
_index_start
=
0
df
.
_index_end
=
df
.
_length_original
df
.
_active_fraction
=
1
return
df

Return a DataFrame where all columns are trimmed by the active range .


def take(self, indices):
        '''Returns a DataFrame containing only rows indexed by indices

        {note_copy}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df.take([0,2])
         #  s      x
         0  a      1
         1  c      3

        :param indices: sequence (list or numpy array) with row numbers
        :return: DataFrame which is a shallow copy of the original data.
        :rtype: DataFrame
        '''
        df = self.copy()
        # if the columns in ds already have a ColumnIndex
        # we could do, direct_indices = df.column['bla'].indices[indices]
        # which should be shared among multiple ColumnIndex'es, so we store
        # them in this dict
        direct_indices_map = {}
        indices = np.array(indices)
        for name in df:
            column = df.columns.get(name)
            if column is not None:
                # we optimize this somewhere, so we don't do multiple
                # levels of indirection
                if isinstance(column, ColumnIndexed):
                    # TODO: think about what happpens when the indices are masked.. ?
                    if id(column.indices) not in direct_indices_map:
                        direct_indices = column.indices[indices]
                        direct_indices_map[id(column.indices)] = direct_indices
                    else:
                        direct_indices = direct_indices_map[id(column.indices)]
                    df.columns[name] = ColumnIndexed(column.df, direct_indices, column.name)
                else:
                    df.columns[name] = ColumnIndexed(self, indices, name)
        df._length_original = len(indices)
        df._length_unfiltered = df._length_original
        df.set_selection(None, name=FILTER_SELECTION_NAME)
        return df


def extract(self):
        '''Return a DataFrame containing only the filtered rows.

        {note_copy}

        The resulting DataFrame may be more efficient to work with when the original DataFrame is
        heavily filtered (contains just a small number of rows).

        If no filtering is applied, it returns a trimmed view.
        For the returned df, len(df) == df.length_original() == df.length_unfiltered()

        :rtype: DataFrame
        '''
        trimmed = self.trim()
        if trimmed.filtered:
            indices = trimmed._filtered_range_to_unfiltered_indices(0, len(trimmed))
            return trimmed.take(indices)
        else:
            return trimmed

def extract ( self ) : trimmed = self . trim ( ) if trimmed . filtered : indices = trimmed . _filtered_range_to_unfiltered_indices ( 0 , len ( trimmed ) ) return trimmed . take ( indices ) else : return trimmed

Return a DataFrame containing only the filtered rows .


def sample(self, n=None, frac=None, replace=False, weights=None, random_state=None):
        '''Returns a DataFrame with a random set of rows

        {note_copy}

        Provide either n or frac.

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df
          #  s      x
          0  a      1
          1  b      2
          2  c      3
          3  d      4
        >>> df.sample(n=2, random_state=42) # 2 random rows, fixed seed
          #  s      x
          0  b      2
          1  d      4
        >>> df.sample(frac=1, random_state=42) # 'shuffling'
          #  s      x
          0  c      3
          1  a      1
          2  d      4
          3  b      2
        >>> df.sample(frac=1, replace=True, random_state=42) # useful for bootstrap (may contain repeated samples)
          #  s      x
          0  d      4
          1  a      1
          2  a      1
          3  d      4

        :param int n: number of samples to take (default 1 if frac is None)
        :param float frac: fractional number of takes to take
        :param bool replace: If true, a row may be drawn multiple times
        :param str or expression weights: (unnormalized) probability that a row can be drawn
        :param int or RandomState: seed or RandomState for reproducability, when None a random seed it chosen
        :return: {return_shallow_copy}
        :rtype: DataFrame
        '''
        self = self.extract()
        if type(random_state) == int or random_state is None:
            random_state = np.random.RandomState(seed=random_state)
        if n is None and frac is None:
            n = 1
        elif frac is not None:
            n = int(round(frac * len(self)))
        weights_values = None
        if weights is not None:
            weights_values = self.evaluate(weights)
            weights_values = weights_values / self.sum(weights)
        indices = random_state.choice(len(self), n, replace=replace, p=weights_values)
        return self.take(indices)

def sample ( self , n = None , frac = None , replace = False , weights = None , random_state = None ) : self = self . extract ( ) if type ( random_state ) == int or random_state is None : random_state = np . random . RandomState ( seed = random_state ) if n is None and frac is None : n = 1 elif frac is not None : n = int ( round ( frac * len ( self ) ) ) weights_values = None if weights is not None : weights_values = self . evaluate ( weights ) weights_values = weights_values / self . sum ( weights ) indices = random_state . choice ( len ( self ) , n , replace = replace , p = weights_values ) return self . take ( indices )


def split_random(self, frac, random_state=None):
        '''Returns a list containing random portions of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex, import numpy as np
        >>> np.random.seed(111)
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split_random(frac=0.3, random_state=42):
        ...     print(dfs.x.values)
        ...
        [8 1 5]
        [0 7 2 9 4 3 6]
        >>> for split in df.split_random(frac=[0.2, 0.3, 0.5], random_state=42):
        ...     print(dfs.x.values)
        [8 1]
        [5 0 7]
        [2 9 4 3 6]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :param int random_state: (default, None) Random number seed for reproducibility.
        :return: A list of DataFrames.
        :rtype: list
        '''
        self = self.extract()
        if type(random_state) == int or random_state is None:
            random_state = np.random.RandomState(seed=random_state)
        indices = random_state.choice(len(self), len(self), replace=False)
        return self.take(indices).split(frac)

def split_random ( self , frac , random_state = None ) : self = self . extract ( ) if type ( random_state ) == int or random_state is None : random_state = np . random . RandomState ( seed = random_state ) indices = random_state . choice ( len ( self ) , len ( self ) , replace = False ) return self . take ( indices ) . split ( frac )

Returns a list containing random portions of the DataFrame .


def split(self, frac):
        '''Returns a list containing ordered subsets of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split(frac=0.3):
        ...     print(dfs.x.values)
        ...
        [0 1 3]
        [3 4 5 6 7 8 9]
        >>> for split in df.split(frac=[0.2, 0.3, 0.5]):
        ...     print(dfs.x.values)
        [0 1]
        [2 3 4]
        [5 6 7 8 9]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :return: A list of DataFrames.
        :rtype: list
        '''
        self = self.extract()
        if _issequence(frac):
            # make sure it is normalized
            total = sum(frac)
            frac = [k / total for k in frac]
        else:
            assert frac <= 1, "fraction should be <= 1"
            frac = [frac, 1 - frac]
        offsets = np.round(np.cumsum(frac) * len(self)).astype(np.int64)
        start = 0
        for offset in offsets:
            yield self[start:offset]
            start = offset

def
split
(
self
,
frac
)
:
self
=
self
.
extract
(
)
if
_issequence
(
frac
)
:
# make sure it is normalized
total
=
sum
(
frac
)
frac
=
[
k
/
total
for
k
in
frac
]
else
:
assert
frac
<=
1
,
"fraction should be <= 1"
frac
=
[
frac
,
1
frac
]
offsets
=
np
.
round
(
np
.
cumsum
(
frac
)
*
len
(
self
)
)
.
astype
(
np
.
int64
)
start
=
0
for
offset
in
offsets
:
yield
self
[
start
:
offset
]
start
=
offset

Returns a list containing ordered subsets of the DataFrame .


def sort(self, by, ascending=True, kind='quicksort'):
        '''Return a sorted DataFrame, sorted by the expression 'by'

        {note_copy}

        {note_filter}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df['y'] = (df.x-1.8)**2
        >>> df
          #  s      x     y
          0  a      1  0.64
          1  b      2  0.04
          2  c      3  1.44
          3  d      4  4.84
        >>> df.sort('y', ascending=False)  # Note: passing '(x-1.8)**2' gives the same result
          #  s      x     y
          0  d      4  4.84
          1  c      3  1.44
          2  a      1  0.64
          3  b      2  0.04

        :param str or expression by: expression to sort by
        :param bool ascending: ascending (default, True) or descending (False)
        :param str kind: kind of algorithm to use (passed to numpy.argsort)
        '''
        self = self.trim()
        values = self.evaluate(by, filtered=False)
        indices = np.argsort(values, kind=kind)
        if not ascending:
            indices = indices[::-1].copy()  # this may be used a lot, so copy for performance
        return self.take(indices)

Return a sorted DataFrame sorted by the expression by


def materialize(self, virtual_column, inplace=False):
        '''Returns a new DataFrame where the virtual column is turned into an in memory numpy array.

        Example:

        >>> x = np.arange(1,4)
        >>> y = np.arange(2,5)
        >>> df = vaex.from_arrays(x=x, y=y)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5 # 'r' is a virtual column (computed on the fly)
        >>> df = df.materialize('r')  # now 'r' is a 'real' column (i.e. a numpy array)

        :param inplace: {inplace}
        '''
        df = self.trim(inplace=inplace)
        virtual_column = _ensure_string_from_expression(virtual_column)
        if virtual_column not in df.virtual_columns:
            raise KeyError('Virtual column not found: %r' % virtual_column)
        ar = df.evaluate(virtual_column, filtered=False)
        del df[virtual_column]
        df.add_column(virtual_column, ar)
        return df

def
materialize
(
self
,
virtual_column
,
inplace
=
False
)
:
df
=
self
.
trim
(
inplace
=
inplace
)
virtual_column
=
_ensure_string_from_expression
(
virtual_column
)
if
virtual_column
not
in
df
.
virtual_columns
:
raise
KeyError
(
'Virtual column not found: %r'
%
virtual_column
)
ar
=
df
.
evaluate
(
virtual_column
,
filtered
=
False
)
del
df
[
virtual_column
]
df
.
add_column
(
virtual_column
,
ar
)
return
df

Returns a new DataFrame where the virtual column is turned into an in memory numpy array .


def selection_undo(self, name="default", executor=None):
        """Undo selection, for the name."""
        logger.debug("undo")
        executor = executor or self.executor
        assert self.selection_can_undo(name=name)
        selection_history = self.selection_histories[name]
        index = self.selection_history_indices[name]
        self.selection_history_indices[name] -= 1
        self.signal_selection_changed.emit(self)
        logger.debug("undo: selection history is %r, index is %r", selection_history, self.selection_history_indices[name])


def selection_redo(self, name="default", executor=None):
        """Redo selection, for the name."""
        logger.debug("redo")
        executor = executor or self.executor
        assert self.selection_can_redo(name=name)
        selection_history = self.selection_histories[name]
        index = self.selection_history_indices[name]
        next = selection_history[index + 1]
        self.selection_history_indices[name] += 1
        self.signal_selection_changed.emit(self)
        logger.debug("redo: selection history is %r, index is %r", selection_history, index)


def selection_can_redo(self, name="default"):
        """Can selection name be redone?"""
        return (self.selection_history_indices[name] + 1) < len(self.selection_histories[name])

def selection_can_redo ( self , name = "default" ) : return ( self . selection_history_indices [ name ] + 1 ) < len ( self . selection_histories [ name ] )


def select(self, boolean_expression, mode="replace", name="default", executor=None):
        """Perform a selection, defined by the boolean expression, and combined with the previous selection using the given mode.

        Selections are recorded in a history tree, per name, undo/redo can be done for them separately.

        :param str boolean_expression: Any valid column expression, with comparison operators
        :param str mode: Possible boolean operator: replace/and/or/xor/subtract
        :param str name: history tree or selection 'slot' to use
        :param executor:
        :return:
        """
        boolean_expression = _ensure_string_from_expression(boolean_expression)
        if boolean_expression is None and not self.has_selection(name=name):
            pass  # we don't want to pollute the history with many None selections
            self.signal_selection_changed.emit(self)  # TODO: unittest want to know, does this make sense?
        else:
            def create(current):
                return selections.SelectionExpression(boolean_expression, current, mode) if boolean_expression else None
            self._selection(create, name)

Perform a selection defined by the boolean expression and combined with the previous selection using the given mode .