def _make_magseries_plot(axes,
                         stimes,
                         smags,
                         serrs,
                         magsarefluxes=False,
                         ms=2.0):
    '''Makes the mag-series plot tile for `checkplot_png` and
    `twolsp_checkplot_png`.

    axes : matplotlib.axes.Axes object
        The Axes object where the generated plot will go.

    stimes,smags,serrs : np.array
        The mag/flux time-series arrays along with associated errors. These
        should all have been run through nan-stripping and sigma-clipping
        beforehand.

    magsarefluxes : bool
        If True, indicates the input time-series is fluxes and not mags so the
        plot y-axis direction and range can be set appropriately.

    ms : float
        The `markersize` kwarg to use when making the mag-series plot.

    Returns
    -------

    Does not return anything, works on the input Axes object directly.

    '''

    scaledplottime = stimes - npmin(stimes)

    axes.plot(scaledplottime,
              smags,
              marker='o',
              ms=ms, ls='None',mew=0,
              color='green',
              rasterized=True)

    # flip y axis for mags
    if not magsarefluxes:
        plot_ylim = axes.get_ylim()
        axes.set_ylim((plot_ylim[1], plot_ylim[0]))

    # set the x axis limit
    axes.set_xlim((npmin(scaledplottime)-1.0,
                   npmax(scaledplottime)+1.0))

    # make a grid
    axes.grid(color='#a9a9a9',
              alpha=0.9,
              zorder=0,
              linewidth=1.0,
              linestyle=':')

    # make the x and y axis labels
    plot_xlabel = 'JD - %.3f' % npmin(stimes)
    if magsarefluxes:
        plot_ylabel = 'flux'
    else:
        plot_ylabel = 'magnitude'

    axes.set_xlabel(plot_xlabel)
    axes.set_ylabel(plot_ylabel)

    # fix the yaxis ticks (turns off offset and uses the full
    # value of the yaxis tick)
    axes.get_yaxis().get_major_formatter().set_useOffset(False)
    axes.get_xaxis().get_major_formatter().set_useOffset(False)

Makes the mag - series plot tile for checkplot_png and twolsp_checkplot_png .


def precess_coordinates(ra, dec,
                        epoch_one, epoch_two,
                        jd=None,
                        mu_ra=0.0,
                        mu_dec=0.0,
                        outscalar=False):
    '''Precesses target coordinates `ra`, `dec` from `epoch_one` to `epoch_two`.

    This takes into account the jd of the observations, as well as the proper
    motion of the target mu_ra, mu_dec. Adapted from J. D. Hartman's
    VARTOOLS/converttime.c [coordprecess].

    Parameters
    ----------

    ra,dec : float
        The equatorial coordinates of the object at `epoch_one` to precess in
        decimal degrees.

    epoch_one : float
        Origin epoch to precess from to target epoch. This is a float, like:
        1985.0, 2000.0, etc.

    epoch_two : float
        Target epoch to precess from origin epoch. This is a float, like:
        2000.0, 2018.0, etc.

    jd : float
        The full Julian date to use along with the propermotions in `mu_ra`, and
        `mu_dec` to handle proper motion along with the coordinate frame
        precession. If one of `jd`, `mu_ra`, or `mu_dec` is missing, the proper
        motion will not be used to calculate the final precessed coordinates.

    mu_ra,mu_dec : float
        The proper motion in mas/yr in right ascension and declination. If these
        are provided along with `jd`, the total proper motion of the object will
        be taken into account to calculate the final precessed coordinates.

    outscalar : bool
        If True, converts the output coordinates from one-element np.arrays to
        scalars.

    Returns
    -------

    precessed_ra, precessed_dec : float
        A tuple of precessed equatorial coordinates in decimal degrees at
        `epoch_two` taking into account proper motion if `jd`, `mu_ra`, and
        `mu_dec` are provided.

    '''

    raproc, decproc = np.radians(ra), np.radians(dec)

    if ((mu_ra != 0.0) and (mu_dec != 0.0) and jd):

        jd_epoch_one = JD2000 + (epoch_one - epoch_two)*365.25
        raproc = (
            raproc +
            (jd - jd_epoch_one)*mu_ra*MAS_P_YR_TO_RAD_P_DAY/np.cos(decproc)
        )
        decproc = decproc + (jd - jd_epoch_one)*mu_dec*MAS_P_YR_TO_RAD_P_DAY

    ca = np.cos(raproc)
    cd = np.cos(decproc)
    sa = np.sin(raproc)
    sd = np.sin(decproc)

    if epoch_one != epoch_two:

        t1 = 1.0e-3 * (epoch_two - epoch_one)
        t2 = 1.0e-3 * (epoch_one - 2000.0)

        a = ( t1*ARCSEC_TO_RADIANS * (23062.181 + t2*(139.656 + 0.0139*t2) +
                                      t1*(30.188 - 0.344*t2+17.998*t1)) )
        b = t1*t1*ARCSEC_TO_RADIANS*(79.280 + 0.410*t2 + 0.205*t1) + a
        c = (
            ARCSEC_TO_RADIANS*t1*(20043.109 - t2*(85.33 + 0.217*t2) +
                                  t1*(-42.665 - 0.217*t2 - 41.833*t2))
        )
        sina, sinb, sinc = np.sin(a), np.sin(b), np.sin(c)
        cosa, cosb, cosc = np.cos(a), np.cos(b), np.cos(c)

        precmatrix = np.matrix([[cosa*cosb*cosc - sina*sinb,
                                 sina*cosb + cosa*sinb*cosc,
                                 cosa*sinc],
                                [-cosa*sinb - sina*cosb*cosc,
                                 cosa*cosb - sina*sinb*cosc,
                                 -sina*sinc],
                                [-cosb*sinc,
                                 -sinb*sinc,
                                 cosc]])

        precmatrix = precmatrix.transpose()

        x = (np.matrix([cd*ca, cd*sa, sd])).transpose()

        x2 = precmatrix * x

        outra = np.arctan2(x2[1],x2[0])
        outdec = np.arcsin(x2[2])

        outradeg = np.rad2deg(outra)
        outdecdeg = np.rad2deg(outdec)

        if outradeg < 0.0:
            outradeg = outradeg + 360.0

        if outscalar:
            return float(outradeg), float(outdecdeg)
        else:
            return outradeg, outdecdeg

    else:

        # if the epochs are the same and no proper motion, this will be the same
        # as the input values. if the epochs are the same, but there IS proper
        # motion (and a given JD), then these will be perturbed from the input
        # values of ra, dec by the appropriate amount of motion
        return np.degrees(raproc), np.degrees(decproc)

Precesses target coordinates ra dec from epoch_one to epoch_two .


def _single_true(iterable):
    '''This returns True if only one True-ish element exists in `iterable`.

    Parameters
    ----------

    iterable : iterable

    Returns
    -------

    bool
        True if only one True-ish element exists in `iterable`. False otherwise.

    '''

    # return True if exactly one true found
    iterator = iter(iterable)

    # consume from "i" until first true or it's exhausted
    has_true = any(iterator)

    # carry on consuming until another true value / exhausted
    has_another_true = any(iterator)

    return has_true and not has_another_true

This returns True if only one True - ish element exists in iterable .


def get_epochs_given_midtimes_and_period(
        t_mid,
        period,
        err_t_mid=None,
        t0_fixed=None,
        t0_percentile=None,
        verbose=False
):
    '''This calculates the future epochs for a transit, given a period and a
    starting epoch

    The equation used is::

        t_mid = period*epoch + t0

    Default behavior if no kwargs are used is to define `t0` as the median
    finite time of the passed `t_mid` array.

    Only one of `err_t_mid` or `t0_fixed` should be passed.

    Parameters
    ----------

    t_mid : np.array
        A np.array of transit mid-time measurements

    period : float
        The period used to calculate epochs, per the equation above. For typical
        use cases, a period precise to ~1e-5 days is sufficient to get correct
        epochs.

    err_t_mid : None or np.array
        If provided, contains the errors of the transit mid-time
        measurements. The zero-point epoch is then set equal to the average of
        the transit times, weighted as `1/err_t_mid^2` . This minimizes the
        covariance between the transit epoch and the period (e.g., Gibson et
        al. 2013). For standard O-C analysis this is the best method.

    t0_fixed : None or float:
        If provided, use this t0 as the starting epoch. (Overrides all others).

    t0_percentile : None or float
        If provided, use this percentile of `t_mid` to define `t0`.

    Returns
    -------

    tuple
        This is the of the form `(integer_epoch_array, t0)`.
        `integer_epoch_array` is an array of integer epochs (float-type),
        of length equal to the number of *finite* mid-times passed.

    '''

    kwargarr = np.array([isinstance(err_t_mid,np.ndarray),
                         t0_fixed,
                         t0_percentile])
    if not _single_true(kwargarr) and not np.all(~kwargarr.astype(bool)):
        raise AssertionError(
            'can have at most one of err_t_mid, t0_fixed, t0_percentile')

    t_mid = t_mid[np.isfinite(t_mid)]
    N_midtimes = len(t_mid)

    if t0_fixed:
        t0 = t0_fixed
    elif isinstance(err_t_mid,np.ndarray):
        # get the weighted average. then round it to the nearest transit epoch.
        t0_avg = np.average(t_mid, weights=1/err_t_mid**2)
        t0_options = np.arange(min(t_mid), max(t_mid)+period, period)
        t0 = t0_options[np.argmin(np.abs(t0_options - t0_avg))]
    else:
        if not t0_percentile:
            # if there are an odd number of times, take the median time as
            # epoch=0.  elif there are an even number of times, take the lower
            # of the two middle times as epoch=0.
            if N_midtimes % 2 == 1:
                t0 = np.median(t_mid)
            else:
                t0 = t_mid[int(N_midtimes/2)]
        else:
            t0 = np.sort(t_mid)[int(N_midtimes*t0_percentile/100)]

    epoch = (t_mid - t0)/period

    # do not convert numpy entries to actual ints, because np.nan is float type
    int_epoch = np.round(epoch, 0)

    if verbose:
        LOGINFO('epochs before rounding')
        LOGINFO('\n{:s}'.format(repr(epoch)))
        LOGINFO('epochs after rounding')
        LOGINFO('\n{:s}'.format(repr(int_epoch)))

    return int_epoch, t0

This calculates the future epochs for a transit given a period and a starting epoch


def jd_to_datetime(jd, returniso=False):
    '''This converts a UTC JD to a Python `datetime` object or ISO date string.

    Parameters
    ----------

    jd : float
        The Julian date measured at UTC.

    returniso : bool
        If False, returns a naive Python `datetime` object corresponding to
        `jd`. If True, returns the ISO format string corresponding to the date
        and time at UTC from `jd`.

    Returns
    -------

    datetime or str
        Depending on the value of `returniso`.

    '''

    tt = astime.Time(jd, format='jd', scale='utc')

    if returniso:
        return tt.iso
    else:
        return tt.datetime

def jd_to_datetime ( jd , returniso = False ) : tt = astime . Time ( jd , format = 'jd' , scale = 'utc' ) if returniso : return tt . iso else : return tt . datetime

This converts a UTC JD to a Python datetime object or ISO date string .


def jd_corr(jd,
            ra, dec,
            obslon=None,
            obslat=None,
            obsalt=None,
            jd_type='bjd'):
    '''Returns BJD_TDB or HJD_TDB for input JD_UTC.

    The equation used is::

        BJD_TDB = JD_UTC + JD_to_TDB_corr + romer_delay

    where:

    - JD_to_TDB_corr is the difference between UTC and TDB JDs
    - romer_delay is the delay caused by finite speed of light from Earth-Sun

    This is based on the code at:

    https://mail.scipy.org/pipermail/astropy/2014-April/003148.html

    Note that this does not correct for:

    1. precession of coordinates if the epoch is not 2000.0
    2. precession of coordinates if the target has a proper motion
    3. Shapiro delay
    4. Einstein delay

    Parameters
    ----------

    jd : float or array-like
        The Julian date(s) measured at UTC.

    ra,dec : float
        The equatorial coordinates of the object in decimal degrees.

    obslon,obslat,obsalt : float or None
        The longitude, latitude of the observatory in decimal degrees and
        altitude of the observatory in meters. If these are not provided, the
        corrected JD will be calculated with respect to the center of the Earth.

    jd_type : {'bjd','hjd'}
        Conversion type to perform, either to Baryocentric Julian Date ('bjd')
        or to Heliocenter Julian Date ('hjd').

    Returns
    -------

    float or np.array
        The converted BJD or HJD.

    '''

    if not HAVEKERNEL:
        LOGERROR('no JPL kernel available, can\'t continue!')
        return

    # Source unit-vector
    ## Assume coordinates in ICRS
    ## Set distance to unit (kilometers)

    # convert the angles to degrees
    rarad = np.radians(ra)
    decrad = np.radians(dec)
    cosra = np.cos(rarad)
    sinra = np.sin(rarad)
    cosdec = np.cos(decrad)
    sindec = np.sin(decrad)

    # this assumes that the target is very far away
    src_unitvector = np.array([cosdec*cosra,cosdec*sinra,sindec])

    # Convert epochs to astropy.time.Time
    ## Assume JD(UTC)
    if (obslon is None) or (obslat is None) or (obsalt is None):
        t = astime.Time(jd, scale='utc', format='jd')
    else:
        t = astime.Time(jd, scale='utc', format='jd',
                        location=('%.5fd' % obslon,
                                  '%.5fd' % obslat,
                                  obsalt))

    # Get Earth-Moon barycenter position
    ## NB: jplephem uses Barycentric Dynamical Time, e.g. JD(TDB)
    ## and gives positions relative to solar system barycenter
    barycenter_earthmoon = jplkernel[0,3].compute(t.tdb.jd)

    # Get Moon position vectors from the center of Earth to the Moon
    # this means we get the following vectors from the ephemerides
    # Earth Barycenter (3) -> Moon (301)
    # Earth Barycenter (3) -> Earth (399)
    # so the final vector is [3,301] - [3,399]
    # units are in km
    moonvector = (jplkernel[3,301].compute(t.tdb.jd) -
                  jplkernel[3,399].compute(t.tdb.jd))

    # Compute Earth position vectors (this is for the center of the earth with
    # respect to the solar system barycenter)
    # all these units are in km
    pos_earth = (barycenter_earthmoon - moonvector * 1.0/(1.0+EMRAT))

    if jd_type == 'bjd':

        # Compute BJD correction
        ## Assume source vectors parallel at Earth and Solar System
        ## Barycenter
        ## i.e. source is at infinity
        # the romer_delay correction is (r.dot.n)/c where:
        # r is the vector from SSB to earth center
        # n is the unit vector from
        correction_seconds = np.dot(pos_earth.T, src_unitvector)/CLIGHT_KPS
        correction_days = correction_seconds/SEC_P_DAY

    elif jd_type == 'hjd':

        # Compute HJD correction via Sun ephemeris

        # this is the position vector of the center of the sun in km
        # Solar System Barycenter (0) -> Sun (10)
        pos_sun = jplkernel[0,10].compute(t.tdb.jd)

        # this is the vector from the center of the sun to the center of the
        # earth
        sun_earth_vec = pos_earth - pos_sun

        # calculate the heliocentric correction
        correction_seconds = np.dot(sun_earth_vec.T, src_unitvector)/CLIGHT_KPS
        correction_days = correction_seconds/SEC_P_DAY

    # TDB is the appropriate time scale for these ephemerides
    new_jd = t.tdb.jd + correction_days

    return new_jd


def _lclist_parallel_worker(task):
    '''This is a parallel worker for makelclist.

    Parameters
    ----------

    task : tuple
        This is a tuple containing the following items:

        task[0] = lcf
        task[1] = columns
        task[2] = lcformat
        task[3] = lcformatdir
        task[4] = lcndetkey

    Returns
    -------

    dict or None
        This contains all of the info for the object processed in this LC read
        operation. If this fails, returns None

    '''

    lcf, columns, lcformat, lcformatdir, lcndetkey = task

    # get the bits needed for lcformat handling
    # NOTE: we re-import things in this worker function because sometimes
    # functions can't be pickled correctly for passing them to worker functions
    # in a processing pool
    try:
        formatinfo = get_lcformat(lcformat,
                                  use_lcformat_dir=lcformatdir)
        if formatinfo:
            (dfileglob, readerfunc,
             dtimecols, dmagcols, derrcols,
             magsarefluxes, normfunc) = formatinfo
        else:
            LOGERROR("can't figure out the light curve format")
            return None
    except Exception as e:
        LOGEXCEPTION("can't figure out the light curve format")
        return None

    # we store the full path of the light curve
    lcobjdict = {'lcfname':os.path.abspath(lcf)}

    try:

        # read the light curve in
        lcdict = readerfunc(lcf)

        # this should handle lists/tuples being returned by readerfunc
        # we assume that the first element is the actual lcdict
        # FIXME: figure out how to not need this assumption
        if ( (isinstance(lcdict, (list, tuple))) and
             (isinstance(lcdict[0], dict)) ):
            lcdict = lcdict[0]

        # insert all of the columns
        for colkey in columns:

            if '.' in colkey:
                getkey = colkey.split('.')
            else:
                getkey = [colkey]

            try:
                thiscolval = _dict_get(lcdict, getkey)
            except Exception as e:
                LOGWARNING('column %s does not exist for %s' %
                           (colkey, lcf))
                thiscolval = np.nan

            # update the lcobjdict with this value
            lcobjdict[getkey[-1]] = thiscolval

    except Exception as e:

        LOGEXCEPTION('could not figure out columns for %s' % lcf)

        # insert all of the columns as nans
        for colkey in columns:

            if '.' in colkey:
                getkey = colkey.split('.')
            else:
                getkey = [colkey]

            thiscolval = np.nan

            # update the lclistdict with this value
            lcobjdict[getkey[-1]] = thiscolval

    # now get the actual ndets; this excludes nans and infs
    for dk in lcndetkey:

        try:

            if '.' in dk:
                getdk = dk.split('.')
            else:
                getdk = [dk]

            ndetcol = _dict_get(lcdict, getdk)
            actualndets = ndetcol[np.isfinite(ndetcol)].size
            lcobjdict['%s.ndet' % getdk[-1]] = actualndets

        except Exception as e:
            lcobjdict['%s.ndet' % getdk[-1]] = np.nan

    return lcobjdict

This is a parallel worker for makelclist .


def _cpinfo_key_worker(task):
    '''This wraps `checkplotlist.checkplot_infokey_worker`.

    This is used to get the correct dtype for each element in retrieved results.

    Parameters
    ----------

    task : tuple
        task[0] = cpfile
        task[1] = keyspeclist (infokeys kwarg from `add_cpinfo_to_lclist`)

    Returns
    -------

    dict
        All of the requested keys from the checkplot are returned along with
        their values in a dict.

    '''

    cpfile, keyspeclist = task

    keystoget = [x[0] for x in keyspeclist]
    nonesubs = [x[-2] for x in keyspeclist]
    nansubs = [x[-1] for x in keyspeclist]

    # reform the keystoget into a list of lists
    for i, k in enumerate(keystoget):

        thisk = k.split('.')
        if sys.version_info[:2] < (3,4):
            thisk = [(int(x) if x.isdigit() else x) for x in thisk]
        else:
            thisk = [(int(x) if x.isdecimal() else x) for x in thisk]

        keystoget[i] = thisk

    # add in the objectid as well to match to the object catalog later
    keystoget.insert(0,['objectid'])
    nonesubs.insert(0, '')
    nansubs.insert(0,'')

    # get all the keys we need
    vals = checkplot_infokey_worker((cpfile, keystoget))

    # if they have some Nones, nans, etc., reform them as expected
    for val, nonesub, nansub, valind in zip(vals, nonesubs,
                                            nansubs, range(len(vals))):

        if val is None:
            outval = nonesub
        elif isinstance(val, float) and not np.isfinite(val):
            outval = nansub
        elif isinstance(val, (list, tuple)):
            outval = ', '.join(val)
        else:
            outval = val

        vals[valind] = outval

    return vals

This wraps checkplotlist . checkplot_infokey_worker .


def handle_change(self, change):
        """ Handle changes from atom ContainerLists """
        op = change['operation']
        if op in 'append':
            self.add(len(change['value']), LatLng(*change['item']))
        elif op == 'insert':
            self.add(change['index'], LatLng(*change['item']))
        elif op == 'extend':
            points = [LatLng(*p) for p in change['items']]
            self.addAll([bridge.encode(c) for c in points])
        elif op == '__setitem__':
            self.set(change['index'], LatLng(*change['newitem']))
        elif op == 'pop':
            self.remove(change['index'])
        else:
            raise NotImplementedError(
                "Unsupported change operation {}".format(op))

def
handle_change
(
self
,
change
)
:
op
=
change
[
'operation'
]
if
op
in
'append'
:
self
.
add
(
len
(
change
[
'value'
]
)
,
LatLng
(
*
change
[
'item'
]
)
)
elif
op
==
'insert'
:
self
.
add
(
change
[
'index'
]
,
LatLng
(
*
change
[
'item'
]
)
)
elif
op
==
'extend'
:
points
=
[
LatLng
(
*
p
)
for
p
in
change
[
'items'
]
]
self
.
addAll
(
[
bridge
.
encode
(
c
)
for
c
in
points
]
)
elif
op
==
'__setitem__'
:
self
.
set
(
change
[
'index'
]
,
LatLng
(
*
change
[
'newitem'
]
)
)
elif
op
==
'pop'
:
self
.
remove
(
change
[
'index'
]
)
else
:
raise
NotImplementedError
(
"Unsupported change operation {}"
.
format
(
op
)
)


def create_widget(self):
        """ Create the underlying widget.

        """
        self.init_options()

        #: Retrieve the actual map
        MapFragment.newInstance(self.options).then(
            self.on_map_fragment_created)

        # Holder for the fragment
        self.widget = FrameLayout(self.get_context())

        # I wrote this a few days ago and already forget how this hack works...
        # lol We can't simply get a map reference using getMapAsync in the
        # return value like we normally do with a normal call function return
        # value. The bridge design was modified to store an object that cannot
        # be decoded normally (via a standard Bridge.Packer) by saving the new
        # object in the cache returning the id of the handler or proxy that
        # invoked it. This way we can manually create a new id and pass that
        # "future reference-able" object as our listener. At which point the
        # bridge will create a reference entry in the cache for us with the of
        # the object we gave it. Once in the cache we can use it like any
        # bridge object we created.
        self.map = GoogleMap(__id__=bridge.generate_id())


def init_options(self):
        """ Initialize the underlying map options.

        """
        self.options = GoogleMapOptions()
        d = self.declaration
        self.set_map_type(d.map_type)
        if d.ambient_mode:
            self.set_ambient_mode(d.ambient_mode)
        if (d.camera_position or d.camera_zoom or
                d.camera_tilt or d.camera_bearing):
            self.update_camera()
        if d.map_bounds:
            self.set_map_bounds(d.map_bounds)
        if not d.show_compass:
            self.set_show_compass(d.show_compass)
        if not d.show_zoom_controls:
            self.set_show_zoom_controls(d.show_zoom_controls)
        if not d.show_toolbar:
            self.set_show_toolbar(d.show_toolbar)
        if d.lite_mode:
            self.set_lite_mode(d.lite_mode)
        if not d.rotate_gestures:
            self.set_rotate_gestures(d.rotate_gestures)
        if not d.scroll_gestures:
            self.set_scroll_gestures(d.scroll_gestures)
        if not d.tilt_gestures:
            self.set_tilt_gestures(d.tilt_gestures)
        if not d.zoom_gestures:
            self.set_zoom_gestures(d.zoom_gestures)
        if d.min_zoom:
            self.set_min_zoom(d.min_zoom)
        if d.max_zoom:
            self.set_max_zoom(d.max_zoom)

def init_options ( self ) : self . options = GoogleMapOptions ( ) d = self . declaration self . set_map_type ( d . map_type ) if d . ambient_mode : self . set_ambient_mode ( d . ambient_mode ) if ( d . camera_position or d . camera_zoom or d . camera_tilt or d . camera_bearing ) : self . update_camera ( ) if d . map_bounds : self . set_map_bounds ( d . map_bounds ) if not d . show_compass : self . set_show_compass ( d . show_compass ) if not d . show_zoom_controls : self . set_show_zoom_controls ( d . show_zoom_controls ) if not d . show_toolbar : self . set_show_toolbar ( d . show_toolbar ) if d . lite_mode : self . set_lite_mode ( d . lite_mode ) if not d . rotate_gestures : self . set_rotate_gestures ( d . rotate_gestures ) if not d . scroll_gestures : self . set_scroll_gestures ( d . scroll_gestures ) if not d . tilt_gestures : self . set_tilt_gestures ( d . tilt_gestures ) if not d . zoom_gestures : self . set_zoom_gestures ( d . zoom_gestures ) if d . min_zoom : self . set_min_zoom ( d . min_zoom ) if d . max_zoom : self . set_max_zoom ( d . max_zoom )


def init_map(self):
        """ Add markers, polys, callouts, etc.."""
        d = self.declaration
        if d.show_location:
            self.set_show_location(d.show_location)
        if d.show_traffic:
            self.set_show_traffic(d.show_traffic)
        if d.show_indoors:
            self.set_show_indoors(d.show_indoors)
        if d.show_buildings:
            self.set_show_buildings(d.show_buildings)

        #: Local ref access is faster
        mapview = self.map
        mid = mapview.getId()

        #: Connect signals
        #: Camera
        mapview.onCameraChange.connect(self.on_camera_changed)
        mapview.onCameraMoveStarted.connect(self.on_camera_move_started)
        mapview.onCameraMoveCanceled.connect(self.on_camera_move_stopped)
        mapview.onCameraIdle.connect(self.on_camera_move_stopped)
        mapview.setOnCameraChangeListener(mid)
        mapview.setOnCameraMoveStartedListener(mid)
        mapview.setOnCameraMoveCanceledListener(mid)
        mapview.setOnCameraIdleListener(mid)

        #: Clicks
        mapview.onMapClick.connect(self.on_map_clicked)
        mapview.setOnMapClickListener(mid)
        mapview.onMapLongClick.connect(self.on_map_long_clicked)
        mapview.setOnMapLongClickListener(mid)

        #: Markers
        mapview.onMarkerClick.connect(self.on_marker_clicked)
        mapview.setOnMarkerClickListener(self.map.getId())
        mapview.onMarkerDragStart.connect(self.on_marker_drag_start)
        mapview.onMarkerDrag.connect(self.on_marker_drag)
        mapview.onMarkerDragEnd.connect(self.on_marker_drag_end)
        mapview.setOnMarkerDragListener(mid)

        #: Info window
        mapview.onInfoWindowClick.connect(self.on_info_window_clicked)
        mapview.onInfoWindowLongClick.connect(self.on_info_window_long_clicked)
        mapview.onInfoWindowClose.connect(self.on_info_window_closed)
        mapview.setOnInfoWindowClickListener(mid)
        mapview.setOnInfoWindowCloseListener(mid)
        mapview.setOnInfoWindowLongClickListener(mid)

        #: Polys
        mapview.onPolygonClick.connect(self.on_poly_clicked)
        mapview.onPolylineClick.connect(self.on_poly_clicked)
        mapview.setOnPolygonClickListener(mid)
        mapview.setOnPolylineClickListener(mid)

        #: Circle
        mapview.onCircleClick.connect(self.on_circle_clicked)
        mapview.setOnCircleClickListener(mid)

def
init_map
(
self
)
:
d
=
self
.
declaration
if
d
.
show_location
:
self
.
set_show_location
(
d
.
show_location
)
if
d
.
show_traffic
:
self
.
set_show_traffic
(
d
.
show_traffic
)
if
d
.
show_indoors
:
self
.
set_show_indoors
(
d
.
show_indoors
)
if
d
.
show_buildings
:
self
.
set_show_buildings
(
d
.
show_buildings
)
#: Local ref access is faster
mapview
=
self
.
map
mid
=
mapview
.
getId
(
)
#: Connect signals
#: Camera
mapview
.
onCameraChange
.
connect
(
self
.
on_camera_changed
)
mapview
.
onCameraMoveStarted
.
connect
(
self
.
on_camera_move_started
)
mapview
.
onCameraMoveCanceled
.
connect
(
self
.
on_camera_move_stopped
)
mapview
.
onCameraIdle
.
connect
(
self
.
on_camera_move_stopped
)
mapview
.
setOnCameraChangeListener
(
mid
)
mapview
.
setOnCameraMoveStartedListener
(
mid
)
mapview
.
setOnCameraMoveCanceledListener
(
mid
)
mapview
.
setOnCameraIdleListener
(
mid
)
#: Clicks
mapview
.
onMapClick
.
connect
(
self
.
on_map_clicked
)
mapview
.
setOnMapClickListener
(
mid
)
mapview
.
onMapLongClick
.
connect
(
self
.
on_map_long_clicked
)
mapview
.
setOnMapLongClickListener
(
mid
)
#: Markers
mapview
.
onMarkerClick
.
connect
(
self
.
on_marker_clicked
)
mapview
.
setOnMarkerClickListener
(
self
.
map
.
getId
(
)
)
mapview
.
onMarkerDragStart
.
connect
(
self
.
on_marker_drag_start
)
mapview
.
onMarkerDrag
.
connect
(
self
.
on_marker_drag
)
mapview
.
onMarkerDragEnd
.
connect
(
self
.
on_marker_drag_end
)
mapview
.
setOnMarkerDragListener
(
mid
)
#: Info window
mapview
.
onInfoWindowClick
.
connect
(
self
.
on_info_window_clicked
)
mapview
.
onInfoWindowLongClick
.
connect
(
self
.
on_info_window_long_clicked
)
mapview
.
onInfoWindowClose
.
connect
(
self
.
on_info_window_closed
)
mapview
.
setOnInfoWindowClickListener
(
mid
)
mapview
.
setOnInfoWindowCloseListener
(
mid
)
mapview
.
setOnInfoWindowLongClickListener
(
mid
)
#: Polys
mapview
.
onPolygonClick
.
connect
(
self
.
on_poly_clicked
)
mapview
.
onPolylineClick
.
connect
(
self
.
on_poly_clicked
)
mapview
.
setOnPolygonClickListener
(
mid
)
mapview
.
setOnPolylineClickListener
(
mid
)
#: Circle
mapview
.
onCircleClick
.
connect
(
self
.
on_circle_clicked
)
mapview
.
setOnCircleClickListener
(
mid
)


def init_info_window_adapter(self):
        """ Initialize the info window adapter. Should only be done if one of
        the markers defines a custom view.
        """
        adapter = self.adapter
        if adapter:
            return  #: Already initialized
        adapter = GoogleMap.InfoWindowAdapter()
        adapter.getInfoContents.connect(self.on_info_window_contents_requested)
        adapter.getInfoWindow.connect(self.on_info_window_requested)
        self.map.setInfoWindowAdapter(adapter)

def
init_info_window_adapter
(
self
)
:
adapter
=
self
.
adapter
if
adapter
:
return
#: Already initialized
adapter
=
GoogleMap
.
InfoWindowAdapter
(
)
adapter
.
getInfoContents
.
connect
(
self
.
on_info_window_contents_requested
)
adapter
.
getInfoWindow
.
connect
(
self
.
on_info_window_requested
)
self
.
map
.
setInfoWindowAdapter
(
adapter
)

Initialize the info window adapter . Should only be done if one of the markers defines a custom view .


def on_map_fragment_created(self, obj_id):
        """ Create the fragment and pull the map reference when it's loaded.
        """
        self.fragment = MapFragment(__id__=obj_id)

        #: Setup callback so we know when the map is ready
        self.map.onMapReady.connect(self.on_map_ready)
        self.fragment.getMapAsync(self.map.getId())

        context = self.get_context()

        def on_transaction(id):
            trans = FragmentTransaction(__id__=id)
            trans.add(self.widget.getId(), self.fragment)
            trans.commit()

        def on_fragment_manager(id):
            fm = FragmentManager(__id__=id)
            fm.beginTransaction().then(on_transaction)

        context.widget.getSupportFragmentManager().then(on_fragment_manager)

Create the fragment and pull the map reference when it s loaded .


def destroy(self):
        """ Remove the marker if it was added to the map when destroying"""
        marker = self.marker
        parent = self.parent()
        if marker:
            if parent:
                del parent.markers[marker.__id__]
            marker.remove()
        super(AndroidMapItemBase, self).destroy()

def destroy ( self ) : marker = self . marker parent = self . parent ( ) if marker : if parent : del parent . markers [ marker . __id__ ] marker . remove ( ) super ( AndroidMapItemBase , self ) . destroy ( )


def child_added(self, child):
        """ If a child is added we have to make sure the map adapter exists """
        if child.widget:
            # TODO: Should we keep count and remove the adapter if not all
            # markers request it?
            self.parent().init_info_window_adapter()
        super(AndroidMapMarker, self).child_added(child)


def on_marker(self, marker):
        """ Convert our options into the actual marker object"""
        mid, pos = marker
        self.marker = Marker(__id__=mid)
        mapview = self.parent()
        # Save ref
        mapview.markers[mid] = self

        # Required so the packer can pass the id
        self.marker.setTag(mid)

        # If we have a child widget we must configure the map to use the
        # custom adapter
        for w in self.child_widgets():
            mapview.init_info_window_adapter()
            break

        d = self.declaration
        if d.show_info:
            self.set_show_info(d.show_info)

        #: Can free the options now
        del self.options


def on_marker(self, mid):
        """ Convert our options into the actual circle object"""
        self.marker = Circle(__id__=mid)
        self.parent().markers[mid] = self

        #: Required so the packer can pass the id
        self.marker.setTag(mid)

        d = self.declaration
        if d.clickable:
            self.set_clickable(d.clickable)

        #: Can free the options now
        del self.options


def fit_transform(self, raw_documents, y=None):
        """ Learn the vocabulary dictionary and return term-document matrix.
        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : array, [n_samples, n_features]
            Document-term matrix.
        """
        documents = super(CountVectorizer, self).fit_transform(
            raw_documents=raw_documents, y=y)
        self.n = len(raw_documents)
        m = (self.transform(raw_documents) > 0).astype(int)
        m = m.sum(axis=0).A1
        self.period_ = m
        self.df_ = m / self.n
        return documents

def fit_transform ( self , raw_documents , y = None ) : documents = super ( CountVectorizer , self ) . fit_transform ( raw_documents = raw_documents , y = y ) self . n = len ( raw_documents ) m = ( self . transform ( raw_documents ) > 0 ) . astype ( int ) m = m . sum ( axis = 0 ) . A1 self . period_ = m self . df_ = m / self . n return documents

Learn the vocabulary dictionary and return term - document matrix . This is equivalent to fit followed by transform but more efficiently implemented .


def data(self, X=None, y=None, sentences=None):
        """
        Add data to flow

        """
        self.X = X
        self.y = y
        self.sentences = sentences

def data ( self , X = None , y = None , sentences = None ) : self . X = X self . y = y self . sentences = sentences


def transform(self, transformer):
        """
        Add transformer to flow and apply transformer to data in flow

        Parameters
        ----------
        transformer : Transformer
            a transformer to transform data
        """
        self.transformers.append(transformer)
        from languageflow.transformer.tagged import TaggedTransformer

        if isinstance(transformer, TaggedTransformer):
            self.X, self.y = transformer.transform(self.sentences)
        if isinstance(transformer, TfidfVectorizer):
            self.X = transformer.fit_transform(self.X)
        if isinstance(transformer, CountVectorizer):
            self.X = transformer.fit_transform(self.X)
        if isinstance(transformer, NumberRemover):
            self.X = transformer.transform(self.X)

        if isinstance(transformer, MultiLabelBinarizer):
            self.y = transformer.fit_transform(self.y)

def transform ( self , transformer ) : self . transformers . append ( transformer ) from languageflow . transformer . tagged import TaggedTransformer if isinstance ( transformer , TaggedTransformer ) : self . X , self . y = transformer . transform ( self . sentences ) if isinstance ( transformer , TfidfVectorizer ) : self . X = transformer . fit_transform ( self . X ) if isinstance ( transformer , CountVectorizer ) : self . X = transformer . fit_transform ( self . X ) if isinstance ( transformer , NumberRemover ) : self . X = transformer . transform ( self . X ) if isinstance ( transformer , MultiLabelBinarizer ) : self . y = transformer . fit_transform ( self . y )


def train(self):
        """
        Train model with transformed data
        """
        for i, model in enumerate(self.models):
            N = [int(i * len(self.y)) for i in self.lc_range]
            for n in N:
                X = self.X[:n]
                y = self.y[:n]
                e = Experiment(X, y, model.estimator, self.scores,
                               self.validation_method)
                e.log_folder = self.log_folder
                e.train()

def train ( self ) : for i , model in enumerate ( self . models ) : N = [ int ( i * len ( self . y ) ) for i in self . lc_range ] for n in N : X = self . X [ : n ] y = self . y [ : n ] e = Experiment ( X , y , model . estimator , self . scores , self . validation_method ) e . log_folder = self . log_folder e . train ( )


def export(self, model_name, export_folder):
        """
        Export model and transformers to export_folder

        Parameters
        ----------
        model_name: string
            name of model to export
        export_folder: string
            folder to store exported model and transformers
        """
        for transformer in self.transformers:
            if isinstance(transformer, MultiLabelBinarizer):
                joblib.dump(transformer,
                            join(export_folder, "label.transformer.bin"),
                            protocol=2)
            if isinstance(transformer, TfidfVectorizer):
                joblib.dump(transformer,
                            join(export_folder, "tfidf.transformer.bin"),
                            protocol=2)
            if isinstance(transformer, CountVectorizer):
                joblib.dump(transformer,
                            join(export_folder, "count.transformer.bin"),
                            protocol=2)
            if isinstance(transformer, NumberRemover):
                joblib.dump(transformer,
                            join(export_folder, "number.transformer.bin"),
                            protocol=2)
        model = [model for model in self.models if model.name == model_name][0]
        e = Experiment(self.X, self.y, model.estimator, None)
        model_filename = join(export_folder, "model.bin")
        e.export(model_filename)

def export ( self , model_name , export_folder ) : for transformer in self . transformers : if isinstance ( transformer , MultiLabelBinarizer ) : joblib . dump ( transformer , join ( export_folder , "label.transformer.bin" ) , protocol = 2 ) if isinstance ( transformer , TfidfVectorizer ) : joblib . dump ( transformer , join ( export_folder , "tfidf.transformer.bin" ) , protocol = 2 ) if isinstance ( transformer , CountVectorizer ) : joblib . dump ( transformer , join ( export_folder , "count.transformer.bin" ) , protocol = 2 ) if isinstance ( transformer , NumberRemover ) : joblib . dump ( transformer , join ( export_folder , "number.transformer.bin" ) , protocol = 2 ) model = [ model for model in self . models if model . name == model_name ] [ 0 ] e = Experiment ( self . X , self . y , model . estimator , None ) model_filename = join ( export_folder , "model.bin" ) e . export ( model_filename )


def fit(self, X, y, coef_init=None, intercept_init=None,
            sample_weight=None):
        """Fit linear model with Stochastic Gradient Descent.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data

        y : numpy array, shape (n_samples,)
            Target values

        coef_init : array, shape (n_classes, n_features)
            The initial coefficients to warm-start the optimization.

        intercept_init : array, shape (n_classes,)
            The initial intercept to warm-start the optimization.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples.
            If not provided, uniform weights are assumed. These weights will
            be multiplied with class_weight (passed through the
            constructor) if class_weight is specified

        Returns
        -------
        self : returns an instance of self.
        """
        super(SGDClassifier, self).fit(X, y, coef_init, intercept_init,
                                       sample_weight)

def fit ( self , X , y , coef_init = None , intercept_init = None , sample_weight = None ) : super ( SGDClassifier , self ) . fit ( X , y , coef_init , intercept_init , sample_weight )

Fit linear model with Stochastic Gradient Descent .


def print_cm(cm, labels, hide_zeroes=False, hide_diagonal=False, hide_threshold=None):
    """pretty print for confusion matrixes"""
    columnwidth = max([len(x) for x in labels] + [5])  # 5 is value length
    empty_cell = " " * columnwidth
    # Print header
    print("    " + empty_cell, end=" ")
    for label in labels:
        print("%{0}s".format(columnwidth) % label, end=" ")
    print()
    # Print rows
    for i, label1 in enumerate(labels):
        print("    %{0}s".format(columnwidth) % label1, end=" ")
        for j in range(len(labels)):
            cell = "%{0}.1f".format(columnwidth) % cm[i, j]
            if hide_zeroes:
                cell = cell if float(cm[i, j]) != 0 else empty_cell
            if hide_diagonal:
                cell = cell if i != j else empty_cell
            if hide_threshold:
                cell = cell if cm[i, j] > hide_threshold else empty_cell
            print(cell, end=" ")
        print()

def
print_cm
(
cm
,
labels
,
hide_zeroes
=
False
,
hide_diagonal
=
False
,
hide_threshold
=
)
:
columnwidth
=
max
(
[
len
(
x
)
for
x
in
labels
]
+
[
5
]
)
# 5 is value length
empty_cell
=
" "
*
columnwidth
# Print header
print
(
"    "
+
empty_cell
,
end
=
" "
)
for
label
in
labels
:
print
(
"%{0}s"
.
format
(
columnwidth
)
%
label
,
end
=
" "
)
print
(
)
# Print rows
for
i
,
label1
in
enumerate
(
labels
)
:
print
(
"    %{0}s"
.
format
(
columnwidth
)
%
label1
,
end
=
" "
)
for
j
in
range
(
len
(
labels
)
)
:
cell
=
"%{0}.1f"
.
format
(
columnwidth
)
%
cm
[
i
,
j
]
if
hide_zeroes
:
cell
=
cell
if
float
(
cm
[
i
,
j
]
)
!=
0
else
empty_cell
if
hide_diagonal
:
cell
=
cell
if
i
!=
j
else
empty_cell
if
hide_threshold
:
cell
=
cell
if
cm
[
i
,
j
]
>
hide_threshold
else
empty_cell
print
(
cell
,
end
=
" "
)
print
(
)


def get_from_cache(url: str, cache_dir: Path = None) -> Path:
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    cache_dir.mkdir(parents=True, exist_ok=True)

    filename = re.sub(r'.+/', '', url)
    # get cache path to put the file
    cache_path = cache_dir / filename
    if cache_path.exists():
        return cache_path

    # make HEAD request to check ETag
    response = requests.head(url)

    if response.status_code != 200:
        if "www.dropbox.com" in url:
            # dropbox return code 301, so we ignore this error
            pass
        else:
            raise IOError("HEAD request failed for url {}".format(url))

    # add ETag to filename if it exists
    # etag = response.headers.get("ETag")

    if not cache_path.exists():
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        fd, temp_filename = tempfile.mkstemp()
        logger.info("%s not found in cache, downloading to %s", url, temp_filename)

        # GET file object
        req = requests.get(url, stream=True)
        content_length = req.headers.get('Content-Length')
        total = int(content_length) if content_length is not None else None
        progress = Tqdm.tqdm(unit="B", total=total)
        with open(temp_filename, 'wb') as temp_file:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk: # filter out keep-alive new chunks
                    progress.update(len(chunk))
                    temp_file.write(chunk)

        progress.close()

        logger.info("copying %s to cache at %s", temp_filename, cache_path)
        shutil.copyfile(temp_filename, str(cache_path))
        logger.info("removing temp file %s", temp_filename)
        os.close(fd)
        os.remove(temp_filename)

    return cache_path

Given a URL look for the corresponding dataset in the local cache . If it s not there download it . Then return the path to the cached file .


def fit(self, X, y):
        """Fit CRF according to X, y

        Parameters
        ----------
        X : list of text
            each item is a text
        y: list
           each item is either a label (in multi class problem) or list of
           labels (in multi label problem)
        """
        trainer = pycrfsuite.Trainer(verbose=True)
        for xseq, yseq in zip(X, y):
            trainer.append(xseq, yseq)

        trainer.set_params(self.params)
        if self.filename:
            filename = self.filename
        else:
            filename = 'model.tmp'
        trainer.train(filename)
        tagger = pycrfsuite.Tagger()
        tagger.open(filename)
        self.estimator = tagger

def fit ( self , X , y ) : trainer = pycrfsuite . Trainer ( verbose = True ) for xseq , yseq in zip ( X , y ) : trainer . append ( xseq , yseq ) trainer . set_params ( self . params ) if self . filename : filename = self . filename else : filename = 'model.tmp' trainer . train ( filename ) tagger = pycrfsuite . Tagger ( ) tagger . open ( filename ) self . estimator = tagger


def predict(self, X):
        """Predict class labels for samples in X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Samples.
        """
        if isinstance(X[0], list):
            return [self.estimator.tag(x) for x in X]
        return self.estimator.tag(X)

def predict ( self , X ) : if isinstance ( X [ 0 ] , list ) : return [ self . estimator . tag ( x ) for x in X ] return self . estimator . tag ( X )


def serve(self, port=62000):
        """ Start LanguageBoard web application

        Parameters
        ----------
        port: int
            port to serve web application
        """

        from http.server import HTTPServer, CGIHTTPRequestHandler
        os.chdir(self.log_folder)

        httpd = HTTPServer(('', port), CGIHTTPRequestHandler)
        print("Starting LanguageBoard on port: " + str(httpd.server_port))
        webbrowser.open('http://0.0.0.0:{}'.format(port))
        httpd.serve_forever()

def
serve
(
self
,
port
=
62000
)
:
from
http
.
server
import
HTTPServer
,
CGIHTTPRequestHandler
os
.
chdir
(
self
.
log_folder
)
httpd
=
HTTPServer
(
(
''
,
port
)
,
CGIHTTPRequestHandler
)
print
(
"Starting LanguageBoard on port: "
+
str
(
httpd
.
server_port
)
)
webbrowser
.
open
(
'http://0.0.0.0:{}'
.
format
(
port
)
)
httpd
.
serve_forever
(
)


def predict(self, X):
        """ In order to obtain the most likely label for a list of text

        Parameters
        ----------
        X : list of string
            Raw texts

        Returns
        -------
        C : list of string
            List labels
        """
        x = X
        if not isinstance(X, list):
            x = [X]
        y = self.estimator.predict(x)
        y = [item[0] for item in y]
        y = [self._remove_prefix(label) for label in y]
        if not isinstance(X, list):
            y = y[0]
        return y

def predict ( self , X ) : x = X if not isinstance ( X , list ) : x = [ X ] y = self . estimator . predict ( x ) y = [ item [ 0 ] for item in y ] y = [ self . _remove_prefix ( label ) for label in y ] if not isinstance ( X , list ) : y = y [ 0 ] return y


def fit(self, X, y):
        """Fit KimCNNClassifier according to X, y

        Parameters
        ----------
        X : list of string
            each item is a raw text
        y : list of string
            each item is a label
        """
        ####################
        # Data Loader
        ####################
        word_vector_transformer = WordVectorTransformer(padding='max')
        X = word_vector_transformer.fit_transform(X)
        X = LongTensor(X)
        self.word_vector_transformer = word_vector_transformer

        y_transformer = LabelEncoder()
        y = y_transformer.fit_transform(y)
        y = torch.from_numpy(y)
        self.y_transformer = y_transformer

        dataset = CategorizedDataset(X, y)
        dataloader = DataLoader(dataset,
                                batch_size=self.batch_size,
                                shuffle=True,
                                num_workers=4)

        ####################
        # Model
        ####################
        KERNEL_SIZES = self.kernel_sizes
        NUM_KERNEL = self.num_kernel
        EMBEDDING_DIM = self.embedding_dim

        model = TextCNN(
            vocab_size=word_vector_transformer.get_vocab_size(),
            embedding_dim=EMBEDDING_DIM,
            output_size=len(self.y_transformer.classes_),
            kernel_sizes=KERNEL_SIZES,
            num_kernel=NUM_KERNEL)
        if USE_CUDA:
            model = model.cuda()

        ####################
        # Train
        ####################
        EPOCH = self.epoch
        LR = self.lr

        loss_function = nn.CrossEntropyLoss()
        optimizer = optim.Adam(model.parameters(), lr=LR)

        for epoch in range(EPOCH):
            losses = []
            for i, data in enumerate(dataloader):
                X, y = data
                X, y = Variable(X), Variable(y)

                optimizer.zero_grad()
                model.train()
                output = model(X)

                loss = loss_function(output, y)
                losses.append(loss.data.tolist()[0])
                loss.backward()

                optimizer.step()

                if i % 100 == 0:
                    print("[%d/%d] mean_loss : %0.2f" % (
                        epoch, EPOCH, np.mean(losses)))
                    losses = []
        self.model = model

def
fit
(
self
,
X
,
y
)
:
####################
# Data Loader
####################
word_vector_transformer
=
WordVectorTransformer
(
padding
=
'max'
)
X
=
word_vector_transformer
.
fit_transform
(
X
)
X
=
LongTensor
(
X
)
self
.
word_vector_transformer
=
word_vector_transformer
y_transformer
=
LabelEncoder
(
)
y
=
y_transformer
.
fit_transform
(
y
)
y
=
torch
.
from_numpy
(
y
)
self
.
y_transformer
=
y_transformer
dataset
=
CategorizedDataset
(
X
,
y
)
dataloader
=
DataLoader
(
dataset
,
batch_size
=
self
.
batch_size
,
shuffle
=
True
,
num_workers
=
4
)
####################
# Model
####################
KERNEL_SIZES
=
self
.
kernel_sizes
NUM_KERNEL
=
self
.
num_kernel
EMBEDDING_DIM
=
self
.
embedding_dim
model
=
TextCNN
(
vocab_size
=
word_vector_transformer
.
get_vocab_size
(
)
,
embedding_dim
=
EMBEDDING_DIM
,
output_size
=
len
(
self
.
y_transformer
.
classes_
)
,
kernel_sizes
=
KERNEL_SIZES
,
num_kernel
=
NUM_KERNEL
)
if
USE_CUDA
:
model
=
model
.
cuda
(
)
####################
# Train
####################
EPOCH
=
self
.
epoch
LR
=
self
.
lr
loss_function
=
nn
.
CrossEntropyLoss
(
)
optimizer
=
optim
.
Adam
(
model
.
parameters
(
)
,
lr
=
LR
)
for
epoch
in
range
(
EPOCH
)
:
losses
=
[
]
for
i
,
data
in
enumerate
(
dataloader
)
:
X
,
y
=
data
X
,
y
=
Variable
(
X
)
,
Variable
(
y
)
optimizer
.
zero_grad
(
)
model
.
train
(
)
output
=
model
(
X
)
loss
=
loss_function
(
output
,
y
)
losses
.
append
(
loss
.
data
.
tolist
(
)
[
0
]
)
loss
.
backward
(
)
optimizer
.
step
(
)
if
i
%
100
==
0
:
print
(
"[%d/%d] mean_loss : %0.2f"
%
(
epoch
,
EPOCH
,
np
.
mean
(
losses
)
)
)
losses
=
[
]
self
.
model
=
model


def config_sources(app, environment, cluster, configs_dirs, app_dir,
                   local=False, build=False):
    """Return the config files for an environment & cluster specific app."""
    sources = [
        # Machine-specific
        (configs_dirs, 'hostname'),
        (configs_dirs, 'hostname-local'),
        (configs_dirs, 'hostname-build'),
        # Global
        (configs_dirs, 'common'),
        # Environment + Cluster
        (configs_dirs, 'common-%s' % environment),
        (configs_dirs, 'common-%s-%s' % (environment, cluster)),
        (configs_dirs, 'common-local'),
        (configs_dirs, 'common-build'),
        # Machine-specific overrides
        (configs_dirs, 'common-overrides'),
        # Application-specific
        ([app_dir], '%s-default' % app),
        ([app_dir], '%s-%s' % (app, environment)),
        ([app_dir], '%s-%s-%s' % (app, environment, cluster)),
        (configs_dirs, app),
        (configs_dirs, '%s-%s' % (app, environment)),
        (configs_dirs, '%s-%s-%s' % (app, environment, cluster)),
        ([app_dir], '%s-local' % app),
        ([app_dir], '%s-build' % app),
        (configs_dirs, '%s-local' % app),
        (configs_dirs, '%s-build' % app),
        # Machine-specific application override
        (configs_dirs, '%s-overrides' % app),
    ]

    # Filter out build sources if not requested
    if not build:
        sources = [source for source in sources
                   if not source[1].endswith('-build')]
        # Filter out local sources if not build and not local
        if not local:
            sources = [source for source in sources
                       if not source[1].endswith('-local')]

    return available_sources(sources)

Return the config files for an environment & cluster specific app .


def available_sources(sources):
    """Yield the sources that are present."""
    for dirs, name in sources:
        for directory in dirs:
            fn = os.path.join(directory, name) + '.py'
            if os.path.isfile(fn):
                yield fn

def available_sources ( sources ) : for dirs , name in sources : for directory in dirs : fn = os . path . join ( directory , name ) + '.py' if os . path . isfile ( fn ) : yield fn


def smush_config(sources, initial=None):
    """Merge the configuration sources and return the resulting DotDict."""
    if initial is None:
        initial = {}
    config = DotDict(initial)

    for fn in sources:
        log.debug('Merging %s', fn)
        mod = get_config_module(fn)
        config = mod.update(config)
        log.debug('Current config:\n%s', json.dumps(config, indent=4,
                                                    cls=LenientJSONEncoder))
    return config

def
smush_config
(
sources
,
initial
=
)
:
if
initial
is
:
initial
=
{
}
config
=
DotDict
(
initial
)
for
fn
in
sources
:
log
.
debug
(
'Merging %s'
,
fn
)
mod
=
get_config_module
(
fn
)
config
=
mod
.
update
(
config
)
log
.
debug
(
'Current config:\n%s'
,
json
.
dumps
(
config
,
indent
=
4
,
cls
=
LenientJSONEncoder
)
)
return
config

Merge the configuration sources and return the resulting DotDict .


def merge_dicts(d1, d2, _path=None):
    """
    Merge dictionary d2 into d1, overriding entries in d1 with values from d2.

    d1 is mutated.

    _path is for internal, recursive use.
    """
    if _path is None:
        _path = ()
    if isinstance(d1, dict) and isinstance(d2, dict):
        for k, v in d2.items():
            if isinstance(v, MissingValue) and v.name is None:
                v.name = '.'.join(_path + (k,))

            if isinstance(v, DeletedValue):
                d1.pop(k, None)
            elif k not in d1:
                if isinstance(v, dict):
                    d1[k] = merge_dicts({}, v, _path + (k,))
                else:
                    d1[k] = v
            else:
                if isinstance(d1[k], dict) and isinstance(v, dict):
                    d1[k] = merge_dicts(d1[k], v, _path + (k,))
                elif isinstance(d1[k], list) and isinstance(v, list):
                    # Lists are only supported as leaves
                    d1[k] += v
                elif isinstance(d1[k], MissingValue):
                    d1[k] = v
                elif d1[k] is None:
                    d1[k] = v
                elif type(d1[k]) == type(v):
                    d1[k] = v
                else:
                    raise TypeError('Refusing to replace a %s with a %s'
                                    % (type(d1[k]), type(v)))
    else:
        raise TypeError('Cannot merge a %s with a %s' % (type(d1), type(d2)))

    return d1

def
merge_dicts
(
d1
,
d2
,
_path
=
)
:
if
_path
is
:
_path
=
(
)
if
isinstance
(
d1
,
dict
)
and
isinstance
(
d2
,
dict
)
:
for
k
,
v
in
d2
.
items
(
)
:
if
isinstance
(
v
,
MissingValue
)
and
v
.
name
is
:
v
.
name
=
'.'
.
join
(
_path
+
(
k
,
)
)
if
isinstance
(
v
,
DeletedValue
)
:
d1
.
pop
(
k
,
)
elif
k
not
in
d1
:
if
isinstance
(
v
,
dict
)
:
d1
[
k
]
=
merge_dicts
(
{
}
,
v
,
_path
+
(
k
,
)
)
else
:
d1
[
k
]
=
v
else
:
if
isinstance
(
d1
[
k
]
,
dict
)
and
isinstance
(
v
,
dict
)
:
d1
[
k
]
=
merge_dicts
(
d1
[
k
]
,
v
,
_path
+
(
k
,
)
)
elif
isinstance
(
d1
[
k
]
,
list
)
and
isinstance
(
v
,
list
)
:
# Lists are only supported as leaves
d1
[
k
]
+=
v
elif
isinstance
(
d1
[
k
]
,
MissingValue
)
:
d1
[
k
]
=
v
elif
d1
[
k
]
is
:
d1
[
k
]
=
v
elif
type
(
d1
[
k
]
)
==
type
(
v
)
:
d1
[
k
]
=
v
else
:
raise
TypeError
(
'Refusing to replace a %s with a %s'
%
(
type
(
d1
[
k
]
)
,
type
(
v
)
)
)
else
:
raise
TypeError
(
'Cannot merge a %s with a %s'
%
(
type
(
d1
)
,
type
(
d2
)
)
)
return
d1

Merge dictionary d2 into d1 overriding entries in d1 with values from d2 .


def filter_dict(unfiltered, filter_keys):
    """Return a subset of a dictionary using the specified keys."""
    filtered = DotDict()
    for k in filter_keys:
        filtered[k] = unfiltered[k]
    return filtered

def filter_dict ( unfiltered , filter_keys ) : filtered = DotDict ( ) for k in filter_keys : filtered [ k ] = unfiltered [ k ] return filtered

Return a subset of a dictionary using the specified keys .


def _convert_item(self, obj):
        """
        Convert obj into a DotDict, or list of DotDict.

        Directly nested lists aren't supported.
        Returns the result
        """
        if isinstance(obj, dict) and not isinstance(obj, DotDict):
            obj = DotDict(obj)
        elif isinstance(obj, list):
            # must mutate and not just reassign, otherwise it will
            # just use original object mutable/immutable
            for i, item in enumerate(obj):
                if isinstance(item, dict) and not isinstance(item, DotDict):
                    obj[i] = DotDict(item)
        return obj

Convert obj into a DotDict or list of DotDict .


def filter_config(config, deploy_config):
    """Return a config subset using the filter defined in the deploy config."""
    if not os.path.isfile(deploy_config):
        return DotDict()
    config_module = get_config_module(deploy_config)
    return config_module.filter(config)

def filter_config ( config , deploy_config ) : if not os . path . isfile ( deploy_config ) : return DotDict ( ) config_module = get_config_module ( deploy_config ) return config_module . filter ( config )

Return a config subset using the filter defined in the deploy config .


def seeded_auth_token(client, service, seed):
    """Return an auth token based on the client+service+seed tuple."""
    hash_func = hashlib.md5()
    token = ','.join((client, service, seed)).encode('utf-8')
    hash_func.update(token)
    return hash_func.hexdigest()

def seeded_auth_token ( client , service , seed ) : hash_func = hashlib . md5 ( ) token = ',' . join ( ( client , service , seed ) ) . encode ( 'utf-8' ) hash_func . update ( token ) return hash_func . hexdigest ( )

Return an auth token based on the client + service + seed tuple .


def write_config(config, app_dir, filename='configuration.json'):
    """Write configuration to the applicaiton directory."""
    path = os.path.join(app_dir, filename)
    with open(path, 'w') as f:
        json.dump(
            config, f, indent=4, cls=DetectMissingEncoder,
            separators=(',', ': '))

def
write_config
(
config
,
app_dir
,
filename
=
'configuration.json'
)
:
path
=
os
.
path
.
join
(
app_dir
,
filename
)
with
open
(
path
,
'w'
)
as
f
:
json
.
dump
(
config
,
f
,
indent
=
4
,
cls
=
DetectMissingEncoder
,
separators
=
(
','
,
': '
)
)

Write configuration to the applicaiton directory .


def validate_date(date_text):
    """Return True if valid, raise ValueError if not"""
    try:
        if int(date_text) < 0:
            return True
    except ValueError:
        pass

    try:
        datetime.strptime(date_text, '%Y-%m-%d')
        return True
    except ValueError:
        pass

    raise ValueError('Dates must be negative integers or YYYY-MM-DD in the past.')

Return True if valid raise ValueError if not


def get_download_total(rows):
    """Return the total downloads, and the downloads column"""
    headers = rows.pop(0)
    index = headers.index('download_count')
    total_downloads = sum(int(row[index]) for row in rows)

    rows.insert(0, headers)
    return total_downloads, index

def get_download_total ( rows ) : headers = rows . pop ( 0 ) index = headers . index ( 'download_count' ) total_downloads = sum ( int ( row [ index ] ) for row in rows ) rows . insert ( 0 , headers ) return total_downloads , index

Return the total downloads and the downloads column


def add_download_total(rows):
    """Add a final row to rows showing the total downloads"""
    total_row = [""] * len(rows[0])
    total_row[0] = "Total"
    total_downloads, downloads_column = get_download_total(rows)
    total_row[downloads_column] = str(total_downloads)
    rows.append(total_row)

    return rows

def add_download_total ( rows ) : total_row = [ "" ] * len ( rows [ 0 ] ) total_row [ 0 ] = "Total" total_downloads , downloads_column = get_download_total ( rows ) total_row [ downloads_column ] = str ( total_downloads ) rows . append ( total_row ) return rows


def find_and_patch_entry(soup, entry):
    """
    Modify soup so Dash.app can generate TOCs on the fly.
    """
    link = soup.find("a", {"class": "headerlink"}, href="#" + entry.anchor)
    tag = soup.new_tag("a")
    tag["name"] = APPLE_REF_TEMPLATE.format(entry.type, entry.name)
    if link:
        link.parent.insert(0, tag)
        return True
    elif entry.anchor.startswith("module-"):
        soup.h1.parent.insert(0, tag)
        return True
    else:
        return False

def find_and_patch_entry ( soup , entry ) : link = soup . find ( "a" , { "class" : "headerlink" } , href = "#" + entry . anchor ) tag = soup . new_tag ( "a" ) tag [ "name" ] = APPLE_REF_TEMPLATE . format ( entry . type , entry . name ) if link : link . parent . insert ( 0 , tag ) return True elif entry . anchor . startswith ( "module-" ) : soup . h1 . parent . insert ( 0 , tag ) return True else : return False

Modify soup so Dash . app can generate TOCs on the fly .


def inv_entry_to_path(data):
    """
    Determine the path from the intersphinx inventory entry

    Discard the anchors between head and tail to make it
    compatible with situations where extra meta information is encoded.
    """
    path_tuple = data[2].split("#")
    if len(path_tuple) > 1:
        path_str = "#".join((path_tuple[0], path_tuple[-1]))
    else:
        path_str = data[2]
    return path_str

def inv_entry_to_path ( data ) : path_tuple = data [ 2 ] . split ( "#" ) if len ( path_tuple ) > 1 : path_str = "#" . join ( ( path_tuple [ 0 ] , path_tuple [ - 1 ] ) ) else : path_str = data [ 2 ] return path_str


def main(
    source,
    force,
    name,
    quiet,
    verbose,
    destination,
    add_to_dash,
    add_to_global,
    icon,
    index_page,
    enable_js,
    online_redirect_url,
    parser,
):
    """
    Convert docs from SOURCE to Dash.app's docset format.
    """
    try:
        logging.config.dictConfig(
            create_log_config(verbose=verbose, quiet=quiet)
        )
    except ValueError as e:
        click.secho(e.args[0], fg="red")
        raise SystemExit(1)

    if icon:
        icon_data = icon.read()
        if not icon_data.startswith(PNG_HEADER):
            log.error(
                '"{}" is not a valid PNG image.'.format(
                    click.format_filename(icon.name)
                )
            )
            raise SystemExit(1)
    else:
        icon_data = None

    source, dest, name = setup_paths(
        source,
        destination,
        name=name,
        add_to_global=add_to_global,
        force=force,
    )
    if parser is None:
        parser = parsers.get_doctype(source)
        if parser is None:
            log.error(
                '"{}" does not contain a known documentation format.'.format(
                    click.format_filename(source)
                )
            )
            raise SystemExit(errno.EINVAL)
    docset = prepare_docset(
        source, dest, name, index_page, enable_js, online_redirect_url
    )
    doc_parser = parser(doc_path=docset.docs)
    log.info(
        (
            "Converting "
            + click.style("{parser_name}", bold=True)
            + ' docs from "{src}" to "{dst}".'
        ).format(
            parser_name=parser.name,
            src=click.format_filename(source, shorten=True),
            dst=click.format_filename(dest),
        )
    )

    with docset.db_conn:
        log.info("Parsing documentation...")
        toc = patch_anchors(doc_parser, show_progressbar=not quiet)
        for entry in doc_parser.parse():
            docset.db_conn.execute(
                "INSERT INTO searchIndex VALUES (NULL, ?, ?, ?)",
                entry.as_tuple(),
            )
            toc.send(entry)
        count = docset.db_conn.execute(
            "SELECT COUNT(1) FROM searchIndex"
        ).fetchone()[0]
        log.info(
            (
                "Added "
                + click.style("{count:,}", fg="green" if count > 0 else "red")
                + " index entries."
            ).format(count=count)
        )
        toc.close()

    if icon_data:
        add_icon(icon_data, dest)

    if add_to_dash or add_to_global:
        log.info("Adding to Dash.app...")
        os.system('open -a dash "{}"'.format(dest))

Convert docs from SOURCE to Dash . app s docset format .


def create_log_config(verbose, quiet):
    """
    We use logging's levels as an easy-to-use verbosity controller.
    """
    if verbose and quiet:
        raise ValueError(
            "Supplying both --quiet and --verbose makes no sense."
        )
    elif verbose:
        level = logging.DEBUG
    elif quiet:
        level = logging.ERROR
    else:
        level = logging.INFO

    logger_cfg = {"handlers": ["click_handler"], "level": level}

    return {
        "version": 1,
        "formatters": {"click_formatter": {"format": "%(message)s"}},
        "handlers": {
            "click_handler": {
                "level": level,
                "class": "doc2dash.__main__.ClickEchoHandler",
                "formatter": "click_formatter",
            }
        },
        "loggers": {"doc2dash": logger_cfg, "__main__": logger_cfg},
    }

We use logging s levels as an easy - to - use verbosity controller .


def setup_paths(source, destination, name, add_to_global, force):
    """
    Determine source and destination using the options.
    """
    if source[-1] == "/":
        source = source[:-1]
    if not name:
        name = os.path.split(source)[-1]
    elif name.endswith(".docset"):
        name = name.replace(".docset", "")
    if add_to_global:
        destination = DEFAULT_DOCSET_PATH
    dest = os.path.join(destination or "", name + ".docset")
    dst_exists = os.path.lexists(dest)
    if dst_exists and force:
        shutil.rmtree(dest)
    elif dst_exists:
        log.error(
            'Destination path "{}" already exists.'.format(
                click.format_filename(dest)
            )
        )
        raise SystemExit(errno.EEXIST)
    return source, dest, name

def
setup_paths
(
source
,
destination
,
name
,
add_to_global
,
force
)
:
if
source
[
1
]
==
"/"
:
source
=
source
[
:
1
]
if
not
name
:
name
=
os
.
path
.
split
(
source
)
[
1
]
elif
name
.
endswith
(
".docset"
)
:
name
=
name
.
replace
(
".docset"
,
""
)
if
add_to_global
:
destination
=
DEFAULT_DOCSET_PATH
dest
=
os
.
path
.
join
(
destination
or
""
,
name
+
".docset"
)
dst_exists
=
os
.
path
.
lexists
(
dest
)
if
dst_exists
and
force
:
shutil
.
rmtree
(
dest
)
elif
dst_exists
:
log
.
error
(
'Destination path "{}" already exists.'
.
format
(
click
.
format_filename
(
dest
)
)
)
raise
SystemExit
(
errno
.
EEXIST
)
return
source
,
dest
,
name

Determine source and destination using the options .


def prepare_docset(
    source, dest, name, index_page, enable_js, online_redirect_url
):
    """
    Create boilerplate files & directories and copy vanilla docs inside.

    Return a tuple of path to resources and connection to sqlite db.
    """
    resources = os.path.join(dest, "Contents", "Resources")
    docs = os.path.join(resources, "Documents")
    os.makedirs(resources)

    db_conn = sqlite3.connect(os.path.join(resources, "docSet.dsidx"))
    db_conn.row_factory = sqlite3.Row
    db_conn.execute(
        "CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, "
        "type TEXT, path TEXT)"
    )
    db_conn.commit()

    plist_path = os.path.join(dest, "Contents", "Info.plist")
    plist_cfg = {
        "CFBundleIdentifier": name,
        "CFBundleName": name,
        "DocSetPlatformFamily": name.lower(),
        "DashDocSetFamily": "python",
        "isDashDocset": True,
        "isJavaScriptEnabled": enable_js,
    }
    if index_page is not None:
        plist_cfg["dashIndexFilePath"] = index_page
    if online_redirect_url is not None:
        plist_cfg["DashDocSetFallbackURL"] = online_redirect_url

    write_plist(plist_cfg, plist_path)

    shutil.copytree(source, docs)

    return DocSet(path=dest, docs=docs, plist=plist_path, db_conn=db_conn)

Create boilerplate files & directories and copy vanilla docs inside .


def add_icon(icon_data, dest):
    """
    Add icon to docset
    """
    with open(os.path.join(dest, "icon.png"), "wb") as f:
        f.write(icon_data)

def add_icon ( icon_data , dest ) : with open ( os . path . join ( dest , "icon.png" ) , "wb" ) as f : f . write ( icon_data )


def run_cell(self, cell):
        """Run the Cell code using the IPython globals and locals

        Args:
            cell (str): Python code to be executed
        """
        globals = self.ipy_shell.user_global_ns
        locals = self.ipy_shell.user_ns
        globals.update({
            "__ipy_scope__": None,
        })
        try:
            with redirect_stdout(self.stdout):
                self.run(cell, globals, locals)
        except:
            self.code_error = True
            if self.options.debug:
                raise BdbQuit
        finally:
            self.finalize()

def run_cell ( self , cell ) : globals = self . ipy_shell . user_global_ns locals = self . ipy_shell . user_ns globals . update ( { "__ipy_scope__" : None , } ) try : with redirect_stdout ( self . stdout ) : self . run ( cell , globals , locals ) except : self . code_error = True if self . options . debug : raise BdbQuit finally : self . finalize ( )


def filter_dict(d, exclude):
    """Return a new dict with specified keys excluded from the origional dict

    Args:
        d (dict): origional dict
        exclude (list): The keys that are excluded
    """
    ret = {}
    for key, value in d.items():
        if key not in exclude:
            ret.update({key: value})
    return ret

def filter_dict ( d , exclude ) : ret = { } for key , value in d . items ( ) : if key not in exclude : ret . update ( { key : value } ) return ret


def redirect_stdout(new_stdout):
    """Redirect the stdout

    Args:
        new_stdout (io.StringIO): New stdout to use instead
    """
    old_stdout, sys.stdout = sys.stdout, new_stdout
    try:
        yield None
    finally:
        sys.stdout = old_stdout

def redirect_stdout ( new_stdout ) : old_stdout , sys . stdout = sys . stdout , new_stdout try : yield None finally : sys . stdout = old_stdout


def format(obj, options):
    """Return a string representation of the Python object

    Args:
        obj: The Python object
        options: Format options
    """
    formatters = {
        float_types: lambda x: '{:.{}g}'.format(x, options.digits),
    }
    for _types, fmtr in formatters.items():
        if isinstance(obj, _types):
            return fmtr(obj)
    try:
        if six.PY2 and isinstance(obj, six.string_types):
            return str(obj.encode('utf-8'))
        return str(obj)
    except:
        return 'OBJECT'

def format ( obj , options ) : formatters = { float_types : lambda x : '{:.{}g}' . format ( x , options . digits ) , } for _types , fmtr in formatters . items ( ) : if isinstance ( obj , _types ) : return fmtr ( obj ) try : if six . PY2 and isinstance ( obj , six . string_types ) : return str ( obj . encode ( 'utf-8' ) ) return str ( obj ) except : return 'OBJECT'


def get_type_info(obj):
    """Get type information for a Python object

    Args:
        obj: The Python object

    Returns:
        tuple: (object type "catagory", object type name)
    """
    if isinstance(obj, primitive_types):
        return ('primitive', type(obj).__name__)
    if isinstance(obj, sequence_types):
        return ('sequence', type(obj).__name__)
    if isinstance(obj, array_types):
        return ('array', type(obj).__name__)
    if isinstance(obj, key_value_types):
        return ('key-value', type(obj).__name__)
    if isinstance(obj, types.ModuleType):
        return ('module', type(obj).__name__)
    if isinstance(obj, (types.FunctionType, types.MethodType)):
        return ('function', type(obj).__name__)
    if isinstance(obj, type):
        if hasattr(obj, '__dict__'):
            return ('class', obj.__name__)
    if isinstance(type(obj), type):
        if hasattr(obj, '__dict__'):
            cls_name = type(obj).__name__
            if cls_name == 'classobj':
                cls_name = obj.__name__
                return ('class', '{}'.format(cls_name))
            if cls_name == 'instance':
                cls_name = obj.__class__.__name__
            return ('instance', '{} instance'.format(cls_name))

    return ('unknown', type(obj).__name__)

def
get_type_info
(
obj
)
:
if
isinstance
(
obj
,
primitive_types
)
:
return
(
'primitive'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
sequence_types
)
:
return
(
'sequence'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
array_types
)
:
return
(
'array'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
key_value_types
)
:
return
(
'key-value'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
types
.
ModuleType
)
:
return
(
'module'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
(
types
.
FunctionType
,
types
.
MethodType
)
)
:
return
(
'function'
,
type
(
obj
)
.
__name__
)
if
isinstance
(
obj
,
type
)
:
if
hasattr
(
obj
,
'__dict__'
)
:
return
(
'class'
,
obj
.
__name__
)
if
isinstance
(
type
(
obj
)
,
type
)
:
if
hasattr
(
obj
,
'__dict__'
)
:
cls_name
=
type
(
obj
)
.
__name__
if
cls_name
==
'classobj'
:
cls_name
=
obj
.
__name__
return
(
'class'
,
'{}'
.
format
(
cls_name
)
)
if
cls_name
==
'instance'
:
cls_name
=
obj
.
__class__
.
__name__
return
(
'instance'
,
'{} instance'
.
format
(
cls_name
)
)
return
(
'unknown'
,
type
(
obj
)
.
__name__
)


def spend_key(self):
        """
        Returns private spend key. None if wallet is view-only.

        :rtype: str or None
        """
        key = self._backend.spend_key()
        if key == numbers.EMPTY_KEY:
            return None
        return key

def spend_key ( self ) : key = self . _backend . spend_key ( ) if key == numbers . EMPTY_KEY : return None return key

Returns private spend key . None if wallet is view - only .


def transfer(self, address, amount,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a transfer from the default account. Returns a list of resulting transactions.

        :param address: destination :class:`Address <monero.address.Address>` or subtype
        :param amount: amount to send
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self.accounts[0].transfer(
                address,
                amount,
                priority=priority,
                payment_id=payment_id,
                unlock_time=unlock_time,
                relay=relay)

def transfer ( self , address , amount , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . accounts [ 0 ] . transfer ( address , amount , priority = priority , payment_id = payment_id , unlock_time = unlock_time , relay = relay )

Sends a transfer from the default account . Returns a list of resulting transactions .


def transfer_multiple(self, destinations,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a batch of transfers from the default account. Returns a list of resulting
        transactions.

        :param destinations: a list of destination and amount pairs: [(address, amount), ...]
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as a destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self.accounts[0].transfer_multiple(
                destinations,
                priority=priority,
                payment_id=payment_id,
                unlock_time=unlock_time,
                relay=relay)

def transfer_multiple ( self , destinations , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . accounts [ 0 ] . transfer_multiple ( destinations , priority = priority , payment_id = payment_id , unlock_time = unlock_time , relay = relay )

Sends a batch of transfers from the default account . Returns a list of resulting transactions .


def balance(self, unlocked=False):
        """
        Returns specified balance.

        :param unlocked: if `True`, return the unlocked balance, otherwise return total balance
        :rtype: Decimal
        """
        return self._backend.balances(account=self.index)[1 if unlocked else 0]

def balance ( self , unlocked = False ) : return self . _backend . balances ( account = self . index ) [ 1 if unlocked else 0 ]


def new_address(self, label=None):
        """
        Creates a new address.

        :param label: address label as `str`
        :rtype: :class:`SubAddress <monero.address.SubAddress>`
        """
        return self._backend.new_address(account=self.index, label=label)

def new_address ( self , label = None ) : return self . _backend . new_address ( account = self . index , label = label )


def transfer(self, address, amount,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a transfer. Returns a list of resulting transactions.

        :param address: destination :class:`Address <monero.address.Address>` or subtype
        :param amount: amount to send
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self._backend.transfer(
            [(address, amount)],
            priority,
            payment_id,
            unlock_time,
            account=self.index,
            relay=relay)

def transfer ( self , address , amount , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . _backend . transfer ( [ ( address , amount ) ] , priority , payment_id , unlock_time , account = self . index , relay = relay )

Sends a transfer . Returns a list of resulting transactions .


def transfer_multiple(self, destinations,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a batch of transfers. Returns a list of resulting transactions.

        :param destinations: a list of destination and amount pairs:
                    [(:class:`Address <monero.address.Address>`, `Decimal`), ...]
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self._backend.transfer(
            destinations,
            priority,
            payment_id,
            unlock_time,
            account=self.index,
            relay=relay)

def transfer_multiple ( self , destinations , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) : return self . _backend . transfer ( destinations , priority , payment_id , unlock_time , account = self . index , relay = relay )

Sends a batch of transfers . Returns a list of resulting transactions .


def to_atomic(amount):
    """Convert Monero decimal to atomic integer of piconero."""
    if not isinstance(amount, (Decimal, float) + _integer_types):
        raise ValueError("Amount '{}' doesn't have numeric type. Only Decimal, int, long and "
                "float (not recommended) are accepted as amounts.")
    return int(amount * 10**12)

Convert Monero decimal to atomic integer of piconero .


def address(addr, label=None):
    """Discover the proper class and return instance for a given Monero address.

    :param addr: the address as a string-like object
    :param label: a label for the address (defaults to `None`)

    :rtype: :class:`Address`, :class:`SubAddress` or :class:`IntegratedAddress`
    """
    addr = str(addr)
    if _ADDR_REGEX.match(addr):
        netbyte = bytearray(unhexlify(base58.decode(addr)))[0]
        if netbyte in Address._valid_netbytes:
            return Address(addr, label=label)
        elif netbyte in SubAddress._valid_netbytes:
            return SubAddress(addr, label=label)
        raise ValueError("Invalid address netbyte {nb:x}. Allowed values are: {allowed}".format(
            nb=netbyte,
            allowed=", ".join(map(
                lambda b: '%02x' % b,
                sorted(Address._valid_netbytes + SubAddress._valid_netbytes)))))
    elif _IADDR_REGEX.match(addr):
        return IntegratedAddress(addr)
    raise ValueError("Address must be either 95 or 106 characters long base58-encoded string, "
        "is {addr} ({len} chars length)".format(addr=addr, len=len(addr)))

Discover the proper class and return instance for a given Monero address .


def with_payment_id(self, payment_id=0):
        """Integrates payment id into the address.

        :param payment_id: int, hexadecimal string or :class:`PaymentID <monero.numbers.PaymentID>`
                    (max 64-bit long)

        :rtype: `IntegratedAddress`
        :raises: `TypeError` if the payment id is too long
        """
        payment_id = numbers.PaymentID(payment_id)
        if not payment_id.is_short():
            raise TypeError("Payment ID {0} has more than 64 bits and cannot be integrated".format(payment_id))
        prefix = 54 if self.is_testnet() else 25 if self.is_stagenet() else 19
        data = bytearray([prefix]) + self._decoded[1:65] + struct.pack('>Q', int(payment_id))
        checksum = bytearray(keccak_256(data).digest()[:4])
        return IntegratedAddress(base58.encode(hexlify(data + checksum)))

Integrates payment id into the address .


def encode(cls, hex):
        """Convert hexadecimal string to mnemonic word representation with checksum.
        """
        out = []
        for i in range(len(hex) // 8):
            word = endian_swap(hex[8*i:8*i+8])
            x = int(word, 16)
            w1 = x % cls.n
            w2 = (x // cls.n + w1) % cls.n
            w3 = (x // cls.n // cls.n + w2) % cls.n
            out += [cls.word_list[w1], cls.word_list[w2], cls.word_list[w3]]
        checksum = cls.get_checksum(" ".join(out))
        out.append(checksum)
        return " ".join(out)

def
encode
(
cls
,
hex
)
:
out
=
[
]
for
i
in
range
(
len
(
hex
)
//
8
)
:
word
=
endian_swap
(
hex
[
8
*
i
:
8
*
i
+
8
]
)
x
=
int
(
word
,
16
)
w1
=
x
%
cls
.
n
w2
=
(
x
//
cls
.
n
+
w1
)
%
cls
.
n
w3
=
(
x
//
cls
.
n
//
cls
.
n
+
w2
)
%
cls
.
n
out
+=
[
cls
.
word_list
[
w1
]
,
cls
.
word_list
[
w2
]
,
cls
.
word_list
[
w3
]
]
checksum
=
cls
.
get_checksum
(
" "
.
join
(
out
)
)
out
.
append
(
checksum
)
return
" "
.
join
(
out
)

Convert hexadecimal string to mnemonic word representation with checksum .


def decode(cls, phrase):
        """Calculate hexadecimal representation of the phrase.
        """
        phrase = phrase.split(" ")
        out = ""
        for i in range(len(phrase) // 3):
            word1, word2, word3 = phrase[3*i:3*i+3]
            w1 = cls.word_list.index(word1)
            w2 = cls.word_list.index(word2) % cls.n
            w3 = cls.word_list.index(word3) % cls.n
            x = w1 + cls.n *((w2 - w1) % cls.n) + cls.n * cls.n * ((w3 - w2) % cls.n)
            out += endian_swap("%08x" % x)
        return out

def
decode
(
cls
,
phrase
)
:
phrase
=
phrase
.
split
(
" "
)
out
=
""
for
i
in
range
(
len
(
phrase
)
//
3
)
:
word1
,
word2
,
word3
=
phrase
[
3
*
i
:
3
*
i
+
3
]
w1
=
cls
.
word_list
.
index
(
word1
)
w2
=
cls
.
word_list
.
index
(
word2
)
%
cls
.
n
w3
=
cls
.
word_list
.
index
(
word3
)
%
cls
.
n
x
=
w1
+
cls
.
n
*
(
(
w2
w1
)
%
cls
.
n
)
+
cls
.
n
*
cls
.
n
*
(
(
w3
w2
)
%
cls
.
n
)
out
+=
endian_swap
(
"%08x"
%
x
)
return
out

Calculate hexadecimal representation of the phrase .


def get_checksum(cls, phrase):
        """Given a mnemonic word string, return a string of the computed checksum.

        :rtype: str
        """
        phrase_split = phrase.split(" ")
        if len(phrase_split) < 12:
            raise ValueError("Invalid mnemonic phrase")
        if len(phrase_split) > 13:
            # Standard format
            phrase = phrase_split[:24]
        else:
            # MyMonero format
            phrase = phrase_split[:12]
        wstr = "".join(word[:cls.unique_prefix_length] for word in phrase)
        wstr = bytearray(wstr.encode('utf-8'))
        z = ((crc32(wstr) & 0xffffffff) ^ 0xffffffff ) >> 0
        z2 = ((z ^ 0xffffffff) >> 0) % len(phrase)
        return phrase_split[z2]

def
get_checksum
(
cls
,
phrase
)
:
phrase_split
=
phrase
.
split
(
" "
)
if
len
(
phrase_split
)
<
12
:
raise
ValueError
(
"Invalid mnemonic phrase"
)
if
len
(
phrase_split
)
>
13
:
# Standard format
phrase
=
phrase_split
[
:
24
]
else
:
# MyMonero format
phrase
=
phrase_split
[
:
12
]
wstr
=
""
.
join
(
word
[
:
cls
.
unique_prefix_length
]
for
word
in
phrase
)
wstr
=
bytearray
(
wstr
.
encode
(
'utf-8'
)
)
z
=
(
(
crc32
(
wstr
)
&
0xffffffff
)
^
0xffffffff
)
>>
0
z2
=
(
(
z
^
0xffffffff
)
>>
0
)
%
len
(
phrase
)
return
phrase_split
[
z2
]

Given a mnemonic word string return a string of the computed checksum .


def one(prompt, *args, **kwargs):
    """Instantiates a picker, registers custom handlers for going back,
    and starts the picker.
    """
    indicator = '‣'
    if sys.version_info < (3, 0):
        indicator = '>'

    def go_back(picker):
        return None, -1

    options, verbose_options = prepare_options(args)
    idx = kwargs.get('idx', 0)

    picker = Picker(verbose_options, title=prompt, indicator=indicator, default_index=idx)
    picker.register_custom_handler(ord('h'), go_back)
    picker.register_custom_handler(curses.KEY_LEFT, go_back)
    with stdout_redirected(sys.stderr):
        option, index = picker.start()
        if index == -1:
            raise QuestionnaireGoBack
        if kwargs.get('return_index', False):
            # `one` was called by a special client, e.g. `many`
            return index
        return options[index]

Instantiates a picker registers custom handlers for going back and starts the picker .


def many(prompt, *args, **kwargs):
    """Calls `pick` in a while loop to allow user to pick many
    options. Returns a list of chosen options.
    """
    def get_options(options, chosen):
        return [options[i] for i, c in enumerate(chosen) if c]

    def get_verbose_options(verbose_options, chosen):
        no, yes = ' ', '✔'
        if sys.version_info < (3, 3):
            no, yes = ' ', '@'
        opts = ['{} {}'.format(yes if c else no, verbose_options[i]) for i, c in enumerate(chosen)]
        return opts + ['{}{}'.format('  ', kwargs.get('done', 'done...'))]

    options, verbose_options = prepare_options(args)
    chosen = [False] * len(options)
    index = kwargs.get('idx', 0)

    default = kwargs.get('default', None)
    if isinstance(default, list):
        for idx in default:
            chosen[idx] = True
    if isinstance(default, int):
        chosen[default] = True

    while True:
        try:
            index = one(prompt, *get_verbose_options(verbose_options, chosen), return_index=True, idx=index)
        except QuestionnaireGoBack:
            if any(chosen):
                raise QuestionnaireGoBack(0)
            else:
                raise QuestionnaireGoBack
        if index == len(options):
            return get_options(options, chosen)
        chosen[index] = not chosen[index]

def
many
(
prompt
,
*
args
,
*
*
kwargs
)
:
def
get_options
(
options
,
chosen
)
:
return
[
options
[
i
]
for
i
,
c
in
enumerate
(
chosen
)
if
c
]
def
get_verbose_options
(
verbose_options
,
chosen
)
:
no
,
yes
=
' '
,
'✔'
if
sys
.
version_info
<
(
3
,
3
)
:
no
,
yes
=
' '
,
'@'
opts
=
[
'{} {}'
.
format
(
yes
if
c
else
no
,
verbose_options
[
i
]
)
for
i
,
c
in
enumerate
(
chosen
)
]
return
opts
+
[
'{}{}'
.
format
(
'  '
,
kwargs
.
get
(
'done'
,
'done...'
)
)
]
options
,
verbose_options
=
prepare_options
(
args
)
chosen
=
[
False
]
*
len
(
options
)
index
=
kwargs
.
get
(
'idx'
,
0
)
default
=
kwargs
.
get
(
'default'
,
)
if
isinstance
(
default
,
list
)
:
for
idx
in
default
:
chosen
[
idx
]
=
True
if
isinstance
(
default
,
int
)
:
chosen
[
default
]
=
True
while
True
:
try
:
index
=
one
(
prompt
,
*
get_verbose_options
(
verbose_options
,
chosen
)
,
return_index
=
True
,
idx
=
index
)
except
QuestionnaireGoBack
:
if
any
(
chosen
)
:
raise
QuestionnaireGoBack
(
0
)
else
:
raise
QuestionnaireGoBack
if
index
==
len
(
options
)
:
return
get_options
(
options
,
chosen
)
chosen
[
index
]
=
not
chosen
[
index
]

Calls pick in a while loop to allow user to pick many options . Returns a list of chosen options .


def prepare_options(options):
    """Create options and verbose options from strings and non-string iterables in
    `options` array.
    """
    options_, verbose_options = [], []
    for option in options:
        if is_string(option):
            options_.append(option)
            verbose_options.append(option)
        else:
            options_.append(option[0])
            verbose_options.append(option[1])
    return options_, verbose_options

def prepare_options ( options ) : options_ , verbose_options = [ ] , [ ] for option in options : if is_string ( option ) : options_ . append ( option ) verbose_options . append ( option ) else : options_ . append ( option [ 0 ] ) verbose_options . append ( option [ 1 ] ) return options_ , verbose_options

Create options and verbose options from strings and non - string iterables in options array .


def raw(prompt, *args, **kwargs):
    """Calls input to allow user to input an arbitrary string. User can go
    back by entering the `go_back` string. Works in both Python 2 and 3.
    """
    go_back = kwargs.get('go_back', '<')
    type_ = kwargs.get('type', str)
    default = kwargs.get('default', '')
    with stdout_redirected(sys.stderr):
        while True:
            try:
                if kwargs.get('secret', False):
                    answer = getpass.getpass(prompt)
                elif sys.version_info < (3, 0):
                    answer = raw_input(prompt)
                else:
                    answer = input(prompt)

                if not answer:
                    answer = default

                if answer == go_back:
                    raise QuestionnaireGoBack
                return type_(answer)
            except ValueError:
                eprint('\n`{}` is not a valid `{}`\n'.format(answer, type_))

def
raw
(
prompt
,
*
args
,
*
*
kwargs
)
:
go_back
=
kwargs
.
get
(
'go_back'
,
'<'
)
type_
=
kwargs
.
get
(
'type'
,
str
)
default
=
kwargs
.
get
(
'default'
,
''
)
with
stdout_redirected
(
sys
.
stderr
)
:
while
True
:
try
:
if
kwargs
.
get
(
'secret'
,
False
)
:
answer
=
getpass
.
getpass
(
prompt
)
elif
sys
.
version_info
<
(
3
,
0
)
:
answer
=
raw_input
(
prompt
)
else
:
answer
=
input
(
prompt
)
if
not
answer
:
answer
=
default
if
answer
==
go_back
:
raise
QuestionnaireGoBack
return
type_
(
answer
)
except
ValueError
:
eprint
(
'\n`{}` is not a valid `{}`\n'
.
format
(
answer
,
type_
)
)

Calls input to allow user to input an arbitrary string . User can go back by entering the go_back string . Works in both Python 2 and 3 .


def get_operator(self, op):
        """Assigns function to the operators property of the instance.
        """
        if op in self.OPERATORS:
            return self.OPERATORS.get(op)
        try:
            n_args = len(inspect.getargspec(op)[0])
            if n_args != 2:
                raise TypeError
        except:
            eprint('Error: invalid operator function. Operators must accept two args.')
            raise
        else:
            return op

Assigns function to the operators property of the instance .


def assign_prompter(self, prompter):
        """If you want to change the core prompters registry, you can
        override this method in a Question subclass.
        """
        if is_string(prompter):
            if prompter not in prompters:
                eprint("Error: '{}' is not a core prompter".format(prompter))
                sys.exit()
            self.prompter = prompters[prompter]
        else:
            self.prompter = prompter

def
assign_prompter
(
self
,
prompter
)
:
if
is_string
(
prompter
)
:
if
prompter
not
in
prompters
:
eprint
(
"Error: '{}' is not a core prompter"
.
format
(
prompter
)
)
sys
.
exit
(
)
self
.
prompter
=
prompters
[
prompter
]
else
:
self
.
prompter
=
prompter

If you want to change the core prompters registry you can override this method in a Question subclass .


def add(self, *args, **kwargs):
        """Add a Question instance to the questions dict. Each key points
        to a list of Question instances with that key. Use the `question`
        kwarg to pass a Question instance if you want, or pass in the same
        args you would pass to instantiate a question.
        """
        if 'question' in kwargs and isinstance(kwargs['question'], Question):
            question = kwargs['question']
        else:
            question = Question(*args, **kwargs)
        self.questions.setdefault(question.key, []).append(question)
        return question

def add ( self , * args , * * kwargs ) : if 'question' in kwargs and isinstance ( kwargs [ 'question' ] , Question ) : question = kwargs [ 'question' ] else : question = Question ( * args , * * kwargs ) self . questions . setdefault ( question . key , [ ] ) . append ( question ) return question

Add a Question instance to the questions dict . Each key points to a list of Question instances with that key . Use the question kwarg to pass a Question instance if you want or pass in the same args you would pass to instantiate a question .