def pref_group(title, prefs, help_text='', static=True, readonly=False):
    """Marks preferences group.

    :param str|unicode title: Group title

    :param list|tuple prefs: Preferences to group.

    :param str|unicode help_text: Field help text.

    :param bool static: Leave this preference static (do not store in DB).

    :param bool readonly: Make this field read only.

    """
    bind_proxy(prefs, title, help_text=help_text, static=static, readonly=readonly)

    for proxy in prefs:  # For preferences already marked by pref().
        if isinstance(proxy, PrefProxy):
            proxy.category = title


def pref(preference, field=None, verbose_name=None, help_text='', static=True, readonly=False):
    """Marks a preference.

    :param preference: Preference variable.

    :param Field field: Django model field to represent this preference.

    :param str|unicode verbose_name: Field verbose name.

    :param str|unicode help_text: Field help text.

    :param bool static: Leave this preference static (do not store in DB).

    :param bool readonly: Make this field read only.

    :rtype: PrefProxy|None
    """
    try:
        bound = bind_proxy(
            (preference,),
            field=field,
            verbose_name=verbose_name,
            help_text=help_text,
            static=static,
            readonly=readonly,
        )
        return bound[0]

    except IndexError:
        return

def pref ( preference , field = None , verbose_name = None , help_text = '' , static = True , readonly = False ) : try : bound = bind_proxy ( ( preference , ) , field = field , verbose_name = verbose_name , help_text = help_text , static = static , readonly = readonly , ) return bound [ 0 ] except IndexError : return


def generate_versionwarning_data_json(app, config=None, **kwargs):
    """
    Generate the ``versionwarning-data.json`` file.

    This file is included in the output and read by the AJAX request when
    accessing to the documentation and used to compare the live versions with
    the curent one.

    Besides, this file contains meta data about the project, the API to use and
    the banner itself.
    """

    # In Sphinx >= 1.8 we use ``config-initied`` signal which comes with the
    # ``config`` object and in Sphinx < 1.8 we use ``builder-initied`` signal
    # that doesn't have the ``config`` object and we take it from the ``app``
    config = config or kwargs.pop('config', None)
    if config is None:
        config = app.config

    if config.versionwarning_project_version in config.versionwarning_messages:
        custom = True
        message = config.versionwarning_messages.get(config.versionwarning_project_version)
    else:
        custom = False
        message = config.versionwarning_default_message

    banner_html = config.versionwarning_banner_html.format(
        id_div=config.versionwarning_banner_id_div,
        banner_title=config.versionwarning_banner_title,
        message=message.format(
            **{config.versionwarning_message_placeholder: ''},
        ),
        admonition_type=config.versionwarning_admonition_type,
    )

    data = json.dumps({
        'meta': {
            'api_url': config.versionwarning_api_url,
        },
        'banner': {
            'html': banner_html,
            'id_div': config.versionwarning_banner_id_div,
            'body_selector': config.versionwarning_body_selector,
            'custom': custom,
        },
        'project': {
            'slug': config.versionwarning_project_slug,
        },
        'version': {
            'slug': config.versionwarning_project_version,
        },
    }, indent=4)

    data_path = os.path.join(STATIC_PATH, 'data')
    if not os.path.exists(data_path):
        os.mkdir(data_path)

    with open(os.path.join(data_path, JSON_DATA_FILENAME), 'w') as f:
        f.write(data)

    # Add the path where ``versionwarning-data.json`` file and
    # ``versionwarning.js`` are saved
    config.html_static_path.append(STATIC_PATH)

Generate the versionwarning - data . json file .


def objective(param_scales=(1, 1), xstar=None, seed=None):
    """Gives objective functions a number of dimensions and parameter range

    Parameters
    ----------
    param_scales : (int, int)
        Scale (std. dev.) for choosing each parameter

    xstar : array_like
        Optimal parameters
    """
    ndim = len(param_scales)

    def decorator(func):

        @wraps(func)
        def wrapper(theta):
            return func(theta)

        def param_init():
            np.random.seed(seed)
            return np.random.randn(ndim,) * np.array(param_scales)

        wrapper.ndim = ndim
        wrapper.param_init = param_init
        wrapper.xstar = xstar

        return wrapper

    return decorator

def objective ( param_scales = ( 1 , 1 ) , xstar = None , seed = None ) : ndim = len ( param_scales ) def decorator ( func ) : @ wraps ( func ) def wrapper ( theta ) : return func ( theta ) def param_init ( ) : np . random . seed ( seed ) return np . random . randn ( ndim , ) * np . array ( param_scales ) wrapper . ndim = ndim wrapper . param_init = param_init wrapper . xstar = xstar return wrapper return decorator


def doublewell(theta):
    """Pointwise minimum of two quadratic bowls"""
    k0, k1, depth = 0.01, 100, 0.5
    shallow = 0.5 * k0 * theta ** 2 + depth
    deep = 0.5 * k1 * theta ** 2
    obj = float(np.minimum(shallow, deep))
    grad = np.where(deep < shallow, k1 * theta, k0 * theta)
    return obj, grad

def doublewell ( theta ) : k0 , k1 , depth = 0.01 , 100 , 0.5 shallow = 0.5 * k0 * theta ** 2 + depth deep = 0.5 * k1 * theta ** 2 obj = float ( np . minimum ( shallow , deep ) ) grad = np . where ( deep < shallow , k1 * theta , k0 * theta ) return obj , grad


def rosenbrock(theta):
    """Objective and gradient for the rosenbrock function"""
    x, y = theta
    obj = (1 - x)**2 + 100 * (y - x**2)**2

    grad = np.zeros(2)
    grad[0] = 2 * x - 400 * (x * y - x**3) - 2
    grad[1] = 200 * (y - x**2)
    return obj, grad

def rosenbrock ( theta ) : x , y = theta obj = ( 1 - x ) ** 2 + 100 * ( y - x ** 2 ) ** 2 grad = np . zeros ( 2 ) grad [ 0 ] = 2 * x - 400 * ( x * y - x ** 3 ) - 2 grad [ 1 ] = 200 * ( y - x ** 2 ) return obj , grad


def booth(theta):
    """Booth's function"""
    x, y = theta

    A = x + 2 * y - 7
    B = 2 * x + y - 5
    obj = A**2 + B**2
    grad = np.array([2 * A + 4 * B, 4 * A + 2 * B])
    return obj, grad

def booth ( theta ) : x , y = theta A = x + 2 * y - 7 B = 2 * x + y - 5 obj = A ** 2 + B ** 2 grad = np . array ( [ 2 * A + 4 * B , 4 * A + 2 * B ] ) return obj , grad


def camel(theta):
    """Three-hump camel function"""
    x, y = theta
    obj = 2 * x ** 2 - 1.05 * x ** 4 + x ** 6 / 6 + x * y + y ** 2
    grad = np.array([
        4 * x - 4.2 * x ** 3 + x ** 5 + y,
        x + 2 * y
    ])
    return obj, grad

def camel ( theta ) : x , y = theta obj = 2 * x ** 2 - 1.05 * x ** 4 + x ** 6 / 6 + x * y + y ** 2 grad = np . array ( [ 4 * x - 4.2 * x ** 3 + x ** 5 + y , x + 2 * y ] ) return obj , grad


def bohachevsky1(theta):
    """One of the Bohachevsky functions"""
    x, y = theta
    obj = x ** 2 + 2 * y ** 2 - 0.3 * np.cos(3 * np.pi * x) - 0.4 * np.cos(4 * np.pi * y) + 0.7
    grad = np.array([
        2 * x + 0.3 * np.sin(3 * np.pi * x) * 3 * np.pi,
        4 * y + 0.4 * np.sin(4 * np.pi * y) * 4 * np.pi,
    ])
    return obj, grad

def bohachevsky1 ( theta ) : x , y = theta obj = x ** 2 + 2 * y ** 2 - 0.3 * np . cos ( 3 * np . pi * x ) - 0.4 * np . cos ( 4 * np . pi * y ) + 0.7 grad = np . array ( [ 2 * x + 0.3 * np . sin ( 3 * np . pi * x ) * 3 * np . pi , 4 * y + 0.4 * np . sin ( 4 * np . pi * y ) * 4 * np . pi , ] ) return obj , grad


def dixon_price(theta):
    """Dixon-Price function"""
    x, y = theta
    obj = (x - 1) ** 2 + 2 * (2 * y ** 2 - x) ** 2
    grad = np.array([
        2 * x - 2 - 4 * (2 * y ** 2 - x),
        16 * (2 * y ** 2 - x) * y,
    ])
    return obj, grad


def styblinski_tang(theta):
    """Styblinski-Tang function"""
    x, y = theta
    obj = 0.5 * (x ** 4 - 16 * x ** 2 + 5 * x + y ** 4 - 16 * y ** 2 + 5 * y)
    grad = np.array([
        2 * x ** 3 - 16 * x + 2.5,
        2 * y ** 3 - 16 * y + 2.5,
    ])
    return obj, grad


def get_all_buckets(self, *args, **kwargs):
        """Return a list of buckets in MimicDB.

        :param boolean force: If true, API call is forced to S3
        """
        if kwargs.pop('force', None):
            buckets = super(S3Connection, self).get_all_buckets(*args, **kwargs)

            for bucket in buckets:
                mimicdb.backend.sadd(tpl.connection, bucket.name)

            return buckets

        return [Bucket(self, bucket) for bucket in mimicdb.backend.smembers(tpl.connection)]

def get_all_buckets ( self , * args , * * kwargs ) : if kwargs . pop ( 'force' , None ) : buckets = super ( S3Connection , self ) . get_all_buckets ( * args , * * kwargs ) for bucket in buckets : mimicdb . backend . sadd ( tpl . connection , bucket . name ) return buckets return [ Bucket ( self , bucket ) for bucket in mimicdb . backend . smembers ( tpl . connection ) ]


def get_bucket(self, bucket_name, validate=True, headers=None, force=None):
        """Return a bucket from MimicDB if it exists. Return a
        S3ResponseError if the bucket does not exist and validate is passed.

        :param boolean force: If true, API call is forced to S3
        """
        if force:
            bucket = super(S3Connection, self).get_bucket(bucket_name, validate, headers)
            mimicdb.backend.sadd(tpl.connection, bucket.name)
            return bucket

        if mimicdb.backend.sismember(tpl.connection, bucket_name):
            return Bucket(self, bucket_name)
        else:
            if validate:
                raise S3ResponseError(404, 'NoSuchBucket')
            else:
                return Bucket(self, bucket_name)

def get_bucket ( self , bucket_name , validate = True , headers = None , force = None ) : if force : bucket = super ( S3Connection , self ) . get_bucket ( bucket_name , validate , headers ) mimicdb . backend . sadd ( tpl . connection , bucket . name ) return bucket if mimicdb . backend . sismember ( tpl . connection , bucket_name ) : return Bucket ( self , bucket_name ) else : if validate : raise S3ResponseError ( 404 , 'NoSuchBucket' ) else : return Bucket ( self , bucket_name )

Return a bucket from MimicDB if it exists . Return a S3ResponseError if the bucket does not exist and validate is passed .


def create_bucket(self, *args, **kwargs):
        """Add the bucket to MimicDB after successful creation.
        """
        bucket = super(S3Connection, self).create_bucket(*args, **kwargs)

        if bucket:
            mimicdb.backend.sadd(tpl.connection, bucket.name)

        return bucket

def create_bucket ( self , * args , * * kwargs ) : bucket = super ( S3Connection , self ) . create_bucket ( * args , * * kwargs ) if bucket : mimicdb . backend . sadd ( tpl . connection , bucket . name ) return bucket

Add the bucket to MimicDB after successful creation .


def sync(self, *buckets):
        """Sync either a list of buckets or the entire connection.

        Force all API calls to S3 and populate the database with the current
        state of S3.

        :param \*string \*buckets: Buckets to sync
        """
        if buckets:
            for _bucket in buckets:
                for key in mimicdb.backend.smembers(tpl.bucket % _bucket):
                    mimicdb.backend.delete(tpl.key % (_bucket, key))

                mimicdb.backend.delete(tpl.bucket % _bucket)

                bucket = self.get_bucket(_bucket, force=True)

                for key in bucket.list(force=True):
                    mimicdb.backend.sadd(tpl.bucket % bucket.name, key.name)
                    mimicdb.backend.hmset(tpl.key % (bucket.name, key.name), dict(size=key.size, md5=key.etag.strip('"')))
        else:
            for bucket in mimicdb.backend.smembers(tpl.connection):
                for key in mimicdb.backend.smembers(tpl.bucket % bucket):
                    mimicdb.backend.delete(tpl.key % (bucket, key))

                mimicdb.backend.delete(tpl.bucket % bucket)

            for bucket in self.get_all_buckets(force=True):
                for key in bucket.list(force=True):
                    mimicdb.backend.sadd(tpl.bucket % bucket.name, key.name)
                    mimicdb.backend.hmset(tpl.key % (bucket.name, key.name), dict(size=key.size, md5=key.etag.strip('"')))

def sync ( self , * buckets ) : if buckets : for _bucket in buckets : for key in mimicdb . backend . smembers ( tpl . bucket % _bucket ) : mimicdb . backend . delete ( tpl . key % ( _bucket , key ) ) mimicdb . backend . delete ( tpl . bucket % _bucket ) bucket = self . get_bucket ( _bucket , force = True ) for key in bucket . list ( force = True ) : mimicdb . backend . sadd ( tpl . bucket % bucket . name , key . name ) mimicdb . backend . hmset ( tpl . key % ( bucket . name , key . name ) , dict ( size = key . size , md5 = key . etag . strip ( '"' ) ) ) else : for bucket in mimicdb . backend . smembers ( tpl . connection ) : for key in mimicdb . backend . smembers ( tpl . bucket % bucket ) : mimicdb . backend . delete ( tpl . key % ( bucket , key ) ) mimicdb . backend . delete ( tpl . bucket % bucket ) for bucket in self . get_all_buckets ( force = True ) : for key in bucket . list ( force = True ) : mimicdb . backend . sadd ( tpl . bucket % bucket . name , key . name ) mimicdb . backend . hmset ( tpl . key % ( bucket . name , key . name ) , dict ( size = key . size , md5 = key . etag . strip ( '"' ) ) )

Sync either a list of buckets or the entire connection .


def get_key(self, *args, **kwargs):
        """Return the key from MimicDB.

        :param boolean force: If true, API call is forced to S3
        """
        if kwargs.pop('force', None):
            headers = kwargs.get('headers', {})
            headers['force'] = True
            kwargs['headers'] = headers

        return super(Bucket, self).get_key(*args, **kwargs)

def get_key ( self , * args , * * kwargs ) : if kwargs . pop ( 'force' , None ) : headers = kwargs . get ( 'headers' , { } ) headers [ 'force' ] = True kwargs [ 'headers' ] = headers return super ( Bucket , self ) . get_key ( * args , * * kwargs )


def _get_key_internal(self, *args, **kwargs):
        """Return None if key is not in the bucket set.

        Pass 'force' in the headers to check S3 for the key, and after fetching
        the key from S3, save the metadata and key to the bucket set.
        """
        if args[1] is not None and 'force' in args[1]:
            key, res = super(Bucket, self)._get_key_internal(*args, **kwargs)

            if key:
                mimicdb.backend.sadd(tpl.bucket % self.name, key.name)
                mimicdb.backend.hmset(tpl.key % (self.name, key.name),
                                    dict(size=key.size,
                                         md5=key.etag.strip('"')))
            return key, res

        key = None

        if mimicdb.backend.sismember(tpl.bucket % self.name, args[0]):
            key = Key(self)
            key.name = args[0]

        return key, None

def _get_key_internal ( self , * args , * * kwargs ) : if args [ 1 ] is not None and 'force' in args [ 1 ] : key , res = super ( Bucket , self ) . _get_key_internal ( * args , * * kwargs ) if key : mimicdb . backend . sadd ( tpl . bucket % self . name , key . name ) mimicdb . backend . hmset ( tpl . key % ( self . name , key . name ) , dict ( size = key . size , md5 = key . etag . strip ( '"' ) ) ) return key , res key = None if mimicdb . backend . sismember ( tpl . bucket % self . name , args [ 0 ] ) : key = Key ( self ) key . name = args [ 0 ] return key , None

Return None if key is not in the bucket set .


def get_all_keys(self, *args, **kwargs):
        """Return a list of keys from MimicDB.

        :param boolean force: If true, API call is forced to S3
        """
        if kwargs.pop('force', None):
            headers = kwargs.get('headers', args[0] if len(args) else None) or dict()
            headers['force'] = True
            kwargs['headers'] = headers

        return super(Bucket, self).get_all_keys(*args, **kwargs)

def get_all_keys ( self , * args , * * kwargs ) : if kwargs . pop ( 'force' , None ) : headers = kwargs . get ( 'headers' , args [ 0 ] if len ( args ) else None ) or dict ( ) headers [ 'force' ] = True kwargs [ 'headers' ] = headers return super ( Bucket , self ) . get_all_keys ( * args , * * kwargs )


def delete_keys(self, *args, **kwargs):
        """Remove each key or key name in an iterable from the bucket set.
        """
        ikeys = iter(kwargs.get('keys', args[0] if args else []))

        while True:
            try:
                key = ikeys.next()
            except StopIteration:
                break

            if isinstance(key, basestring):
                mimicdb.backend.srem(tpl.bucket % self.name, key)
                mimicdb.backend.delete(tpl.key % (self.name, key))
            elif isinstance(key, BotoKey) or isinstance(key, Key):
                mimicdb.backend.srem(tpl.bucket % self.name, key.name)
                mimicdb.backend.delete(tpl.key % (self.name, key.name))

        return super(Bucket, self).delete_keys(*args, **kwargs)

def delete_keys ( self , * args , * * kwargs ) : ikeys = iter ( kwargs . get ( 'keys' , args [ 0 ] if args else [ ] ) ) while True : try : key = ikeys . next ( ) except StopIteration : break if isinstance ( key , basestring ) : mimicdb . backend . srem ( tpl . bucket % self . name , key ) mimicdb . backend . delete ( tpl . key % ( self . name , key ) ) elif isinstance ( key , BotoKey ) or isinstance ( key , Key ) : mimicdb . backend . srem ( tpl . bucket % self . name , key . name ) mimicdb . backend . delete ( tpl . key % ( self . name , key . name ) ) return super ( Bucket , self ) . delete_keys ( * args , * * kwargs )

Remove each key or key name in an iterable from the bucket set .


def _delete_key_internal(self, *args, **kwargs):
        """Remove key name from bucket set.
        """
        mimicdb.backend.srem(tpl.bucket % self.name, args[0])
        mimicdb.backend.delete(tpl.key % (self.name, args[0]))

        return super(Bucket, self)._delete_key_internal(*args, **kwargs)

def _delete_key_internal ( self , * args , * * kwargs ) : mimicdb . backend . srem ( tpl . bucket % self . name , args [ 0 ] ) mimicdb . backend . delete ( tpl . key % ( self . name , args [ 0 ] ) ) return super ( Bucket , self ) . _delete_key_internal ( * args , * * kwargs )


def list(self, *args, **kwargs):
        """Return an iterable of keys from MimicDB.

        :param boolean force: If true, API call is forced to S3
        """
        if kwargs.pop('force', None):
            headers = kwargs.get('headers', args[4] if len(args) > 4 else None) or dict()
            headers['force'] = True
            kwargs['headers'] = headers

            for key in super(Bucket, self).list(*args, **kwargs):
                yield key

        else:
            prefix = kwargs.get('prefix', args[0] if args else '')

            for key in mimicdb.backend.smembers(tpl.bucket % self.name):
                if key.startswith(prefix):
                    k = Key(self, key)

                    meta = mimicdb.backend.hgetall(tpl.key % (self.name, key))

                    if meta:
                        k._load_meta(meta['size'], meta['md5'])

                    yield k

def list ( self , * args , * * kwargs ) : if kwargs . pop ( 'force' , None ) : headers = kwargs . get ( 'headers' , args [ 4 ] if len ( args ) > 4 else None ) or dict ( ) headers [ 'force' ] = True kwargs [ 'headers' ] = headers for key in super ( Bucket , self ) . list ( * args , * * kwargs ) : yield key else : prefix = kwargs . get ( 'prefix' , args [ 0 ] if args else '' ) for key in mimicdb . backend . smembers ( tpl . bucket % self . name ) : if key . startswith ( prefix ) : k = Key ( self , key ) meta = mimicdb . backend . hgetall ( tpl . key % ( self . name , key ) ) if meta : k . _load_meta ( meta [ 'size' ] , meta [ 'md5' ] ) yield k

Return an iterable of keys from MimicDB .


def sync(self):
        """Sync a bucket.

        Force all API calls to S3 and populate the database with the current state of S3.
        """
        for key in mimicdb.backend.smembers(tpl.bucket % self.name):
            mimicdb.backend.delete(tpl.key % (self.name, key))

        mimicdb.backend.delete(tpl.bucket % self.name)
        mimicdb.backend.sadd(tpl.connection, self.name)

        for key in self.list(force=True):
            mimicdb.backend.sadd(tpl.bucket % self.name, key.name)
            mimicdb.backend.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"')))

def sync ( self ) : for key in mimicdb . backend . smembers ( tpl . bucket % self . name ) : mimicdb . backend . delete ( tpl . key % ( self . name , key ) ) mimicdb . backend . delete ( tpl . bucket % self . name ) mimicdb . backend . sadd ( tpl . connection , self . name ) for key in self . list ( force = True ) : mimicdb . backend . sadd ( tpl . bucket % self . name , key . name ) mimicdb . backend . hmset ( tpl . key % ( self . name , key . name ) , dict ( size = key . size , md5 = key . etag . strip ( '"' ) ) )


def lbfgs(x, rho, f_df, maxiter=20):
    """
    Minimize the proximal operator of a given objective using L-BFGS

    Parameters
    ----------
    f_df : function
        Returns the objective and gradient of the function to minimize

    maxiter : int
        Maximum number of L-BFGS iterations
    """

    def f_df_augmented(theta):
        f, df = f_df(theta)
        obj = f + (rho / 2.) * np.linalg.norm(theta - x) ** 2
        grad = df + rho * (theta - x)
        return obj, grad

    res = scipy_minimize(f_df_augmented, x, jac=True, method='L-BFGS-B',
                         options={'maxiter': maxiter, 'disp': False})

    return res.x

def lbfgs ( x , rho , f_df , maxiter = 20 ) : def f_df_augmented ( theta ) : f , df = f_df ( theta ) obj = f + ( rho / 2. ) * np . linalg . norm ( theta - x ) ** 2 grad = df + rho * ( theta - x ) return obj , grad res = scipy_minimize ( f_df_augmented , x , jac = True , method = 'L-BFGS-B' , options = { 'maxiter' : maxiter , 'disp' : False } ) return res . x

Minimize the proximal operator of a given objective using L - BFGS


def smooth(x, rho, penalty, axis=0, newshape=None):
    """
    Applies a smoothing operator along one dimension

    currently only accepts a matrix as input

    Parameters
    ----------
    penalty : float

    axis : int, optional
        Axis along which to apply the smoothing (Default: 0)

    newshape : tuple, optional
        Desired shape of the parameters to apply the nuclear norm to. The given
        parameters are reshaped to an array with this shape, or not reshaped if
        the value of newshape is None. (Default: None)
    """

    orig_shape = x.shape

    if newshape is not None:
        x = x.reshape(newshape)

    # Apply Laplacian smoothing (l2 norm on the parameters multiplied by
    # the laplacian)
    n = x.shape[axis]
    lap_op = spdiags([(2 + rho / penalty) * np.ones(n),
                      -1 * np.ones(n), -1 * np.ones(n)],
                     [0, -1, 1], n, n, format='csc')

    A = penalty * lap_op
    b = rho * np.rollaxis(x, axis, 0)
    return np.rollaxis(spsolve(A, b), axis, 0).reshape(orig_shape)


def sdcone(x, rho):
    """Projection onto the semidefinite cone"""
    U, V = np.linalg.eigh(x)
    return V.dot(np.diag(np.maximum(U, 0)).dot(V.T))

def sdcone ( x , rho ) : U , V = np . linalg . eigh ( x ) return V . dot ( np . diag ( np . maximum ( U , 0 ) ) . dot ( V . T ) )


def simplex(x, rho):
    """
    Projection onto the probability simplex

    http://arxiv.org/pdf/1309.1541v1.pdf
    """

    # sort the elements in descending order
    u = np.flipud(np.sort(x.ravel()))
    lambdas = (1 - np.cumsum(u)) / (1. + np.arange(u.size))
    ix = np.where(u + lambdas > 0)[0].max()
    return np.maximum(x + lambdas[ix], 0)

def
simplex
(
x
,
rho
)
:
# sort the elements in descending order
u
=
np
.
flipud
(
np
.
sort
(
x
.
ravel
(
)
)
)
lambdas
=
(
1
np
.
cumsum
(
u
)
)
/
(
1.
+
np
.
arange
(
u
.
size
)
)
ix
=
np
.
where
(
u
+
lambdas
>
0
)
[
0
]
.
max
(
)
return
np
.
maximum
(
x
+
lambdas
[
ix
]
,
0
)


def columns(x, rho, proxop):
    """Applies a proximal operator to the columns of a matrix"""

    xnext = np.zeros_like(x)

    for ix in range(x.shape[1]):
        xnext[:, ix] = proxop(x[:, ix], rho)

    return xnext

def columns ( x , rho , proxop ) : xnext = np . zeros_like ( x ) for ix in range ( x . shape [ 1 ] ) : xnext [ : , ix ] = proxop ( x [ : , ix ] , rho ) return xnext


def gradient_optimizer(coro):
    """Turns a coroutine into a gradient based optimizer."""

    class GradientOptimizer(Optimizer):

        @wraps(coro)
        def __init__(self, *args, **kwargs):
            self.algorithm = coro(*args, **kwargs)
            self.algorithm.send(None)
            self.operators = []

        def set_transform(self, func):
            self.transform = compose(destruct, func, self.restruct)

        def minimize(self, f_df, x0, display=sys.stdout, maxiter=1e3):

            self.display = display
            self.theta = x0

            # setup
            xk = self.algorithm.send(destruct(x0).copy())
            store = defaultdict(list)
            runtimes = []
            if len(self.operators) == 0:
                self.operators = [proxops.identity()]

            # setup
            obj, grad = wrap(f_df, x0)
            transform = compose(destruct, *reversed(self.operators), self.restruct)

            self.optional_print(tp.header(['Iteration', 'Objective', '||Grad||', 'Runtime']))
            try:
                for k in count():

                    # setup
                    tstart = perf_counter()
                    f = obj(xk)
                    df = grad(xk)
                    xk = transform(self.algorithm.send(df))
                    runtimes.append(perf_counter() - tstart)
                    store['f'].append(f)

                    # Update display
                    self.optional_print(tp.row([k,
                                                f,
                                                np.linalg.norm(destruct(df)),
                                                tp.humantime(runtimes[-1])]))

                    if k >= maxiter:
                        break

            except KeyboardInterrupt:
                pass

            self.optional_print(tp.bottom(4))

            # cleanup
            self.optional_print(u'\u279b Final objective: {}'.format(store['f'][-1]))
            self.optional_print(u'\u279b Total runtime: {}'.format(tp.humantime(sum(runtimes))))
            self.optional_print(u'\u279b Per iteration runtime: {} +/- {}'.format(
                tp.humantime(np.mean(runtimes)),
                tp.humantime(np.std(runtimes)),
            ))

            # result
            return OptimizeResult({
                'x': self.restruct(xk),
                'f': f,
                'df': self.restruct(df),
                'k': k,
                'obj': np.array(store['f']),
            })

    return GradientOptimizer

Turns a coroutine into a gradient based optimizer .


def add(self, operator, *args):
        """Adds a proximal operator to the list of operators"""

        if isinstance(operator, str):
            op = getattr(proxops, operator)(*args)
        elif isinstance(operator, proxops.ProximalOperatorBaseClass):
            op = operator
        else:
            raise ValueError("operator must be a string or a subclass of ProximalOperator")

        self.operators.append(op)
        return self


def _load_meta(self, size, md5):
        """Set key attributes to retrived metadata. Might be extended in the
        future to support more attributes.
        """
        if not hasattr(self, 'local_hashes'):
            self.local_hashes = {}

        self.size = int(size)

        if (re.match('^[a-fA-F0-9]{32}$', md5)):
            self.md5 = md5

def _load_meta ( self , size , md5 ) : if not hasattr ( self , 'local_hashes' ) : self . local_hashes = { } self . size = int ( size ) if ( re . match ( '^[a-fA-F0-9]{32}$' , md5 ) ) : self . md5 = md5

Set key attributes to retrived metadata . Might be extended in the future to support more attributes .


def _send_file_internal(self, *args, **kwargs):
        """Called internally for any type of upload. After upload finishes,
        make sure the key is in the bucket set and save the metadata.
        """
        super(Key, self)._send_file_internal(*args, **kwargs)

        mimicdb.backend.sadd(tpl.bucket % self.bucket.name, self.name)
        mimicdb.backend.hmset(tpl.key % (self.bucket.name, self.name),
                            dict(size=self.size, md5=self.md5))

def _send_file_internal ( self , * args , * * kwargs ) : super ( Key , self ) . _send_file_internal ( * args , * * kwargs ) mimicdb . backend . sadd ( tpl . bucket % self . bucket . name , self . name ) mimicdb . backend . hmset ( tpl . key % ( self . bucket . name , self . name ) , dict ( size = self . size , md5 = self . md5 ) )

Called internally for any type of upload . After upload finishes make sure the key is in the bucket set and save the metadata .


def wrap(f_df, xref, size=1):
    """
    Memoizes an objective + gradient function, and splits it into
    two functions that return just the objective and gradient, respectively.

    Parameters
    ----------
    f_df : function
        Must be unary (takes a single argument)

    xref : list, dict, or array_like
        The form of the parameters

    size : int, optional
        Size of the cache (Default=1)
    """
    memoized_f_df = lrucache(lambda x: f_df(restruct(x, xref)), size)
    objective = compose(first, memoized_f_df)
    gradient = compose(destruct, second, memoized_f_df)
    return objective, gradient

def wrap ( f_df , xref , size = 1 ) : memoized_f_df = lrucache ( lambda x : f_df ( restruct ( x , xref ) ) , size ) objective = compose ( first , memoized_f_df ) gradient = compose ( destruct , second , memoized_f_df ) return objective , gradient

Memoizes an objective + gradient function and splits it into two functions that return just the objective and gradient respectively .


def docstring(docstr):
    """
    Decorates a function with the given docstring

    Parameters
    ----------
    docstr : string
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)
        wrapper.__doc__ = docstr
        return wrapper
    return decorator

def docstring ( docstr ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , * * kwargs ) : return func ( * args , * * kwargs ) wrapper . __doc__ = docstr return wrapper return decorator


def check_grad(f_df, xref, stepsize=1e-6, tol=1e-6, width=15, style='round', out=sys.stdout):
    """
    Compares the numerical gradient to the analytic gradient

    Parameters
    ----------
    f_df : function
        The analytic objective and gradient function to check

    x0 : array_like
        Parameter values to check the gradient at

    stepsize : float, optional
        Stepsize for the numerical gradient. Too big and this will poorly estimate the gradient.
        Too small and you will run into precision issues (default: 1e-6)

    tol : float, optional
        Tolerance to use when coloring correct/incorrect gradients (default: 1e-5)

    width : int, optional
        Width of the table columns (default: 15)

    style : string, optional
        Style of the printed table, see tableprint for a list of styles (default: 'round')
    """
    CORRECT = u'\x1b[32m\N{CHECK MARK}\x1b[0m'
    INCORRECT = u'\x1b[31m\N{BALLOT X}\x1b[0m'

    obj, grad = wrap(f_df, xref, size=0)
    x0 = destruct(xref)
    df = grad(x0)

    # header
    out.write(tp.header(["Numerical", "Analytic", "Error"], width=width, style=style) + "\n")
    out.flush()

    # helper function to parse a number
    def parse_error(number):

        # colors
        failure = "\033[91m"
        passing = "\033[92m"
        warning = "\033[93m"
        end = "\033[0m"
        base = "{}{:0.3e}{}"

        # correct
        if error < 0.1 * tol:
            return base.format(passing, error, end)

        # warning
        elif error < tol:
            return base.format(warning, error, end)

        # failure
        else:
            return base.format(failure, error, end)

    # check each dimension
    num_errors = 0
    for j in range(x0.size):

        # take a small step in one dimension
        dx = np.zeros(x0.size)
        dx[j] = stepsize

        # compute the centered difference formula
        df_approx = (obj(x0 + dx) - obj(x0 - dx)) / (2 * stepsize)
        df_analytic = df[j]

        # absolute error
        abs_error = np.linalg.norm(df_approx - df_analytic)

        # relative error
        error = abs_error if np.allclose(abs_error, 0) else abs_error / \
            (np.linalg.norm(df_analytic) + np.linalg.norm(df_approx))

        num_errors += error >= tol
        errstr = CORRECT if error < tol else INCORRECT
        out.write(tp.row([df_approx, df_analytic, parse_error(error) + ' ' + errstr],
                         width=width, style=style) + "\n")
        out.flush()

    out.write(tp.bottom(3, width=width, style=style) + "\n")
    return num_errors


def evaluate(self, repo, spec, args):
        """
        Evaluate the files identified for checksum.
        """

        status = []

        # Do we have to any thing at all?
        if len(spec['files']) == 0:
            return status

        with cd(repo.rootdir):

            rules = None
            if 'rules-files' in spec and len(spec['rules-files']) > 0:
                rulesfiles = spec['rules-files']
                rules = {}
                for f in rulesfiles:
                    d = json.loads(open(f).read())
                    rules.update(d)
            elif 'rules' in spec:
                rules = {
                    'inline': spec['rules']
                }

            if rules is None or len(rules) == 0:
                print("Regression quality validation has been enabled but no rules file has been specified")
                print("Example: { 'min-r2': 0.25 }. Put this either in file or in dgit.json")
                raise InvalidParameters("Regression quality checking rules missing")

            files = dict([(f, open(f).read()) for f in spec['files']])

            for r in rules:
                if 'min-r2' not in rules[r]:
                    continue
                minr2 = float(rules[r]['min-r2'])
                for f in files:
                    match = re.search(r"R-squared:\s+(\d.\d+)", files[f])
                    if match is None:
                        status.append({
                            'target': f,
                            'validator': self.name,
                            'description': self.description,
                            'rules': r,
                            'status': "ERROR",
                            'message': "Invalid model output"
                            })
                    else:
                        r2 = match.group(1)
                        r2 = float(r2)
                        if r2 > minr2:
                            status.append({
                                'target': f,
                                'validator': self.name,
                                'description': self.description,
                                'rules': r,
                                'status': "OK",
                                'message': "Acceptable R2"
                            })
                        else:
                            status.append({
                                'target': f,
                                'validator': self.name,
                                'description': self.description,
                                'rules': r,
                                'status': "ERROR",
                                'message': "R2 is too low"
                            })

        return status

Evaluate the files identified for checksum .


def evaluate(self, repo, spec, args):
        """
        Check the integrity of the datapackage.json
        """

        status = []
        with cd(repo.rootdir):
            files = spec.get('files', ['*'])
            resource_files = repo.find_matching_files(files)
            files = glob2.glob("**/*")
            disk_files = [f for f in files if os.path.isfile(f) and f != "datapackage.json"]

            allfiles = list(set(resource_files + disk_files))
            allfiles.sort()

            for f in allfiles:
                if f in resource_files and f in disk_files:
                    r = repo.get_resource(f)
                    coded_sha256 = r['sha256']
                    computed_sha256 = compute_sha256(f)
                    if computed_sha256 != coded_sha256:
                        status.append({
                            'target': f,
                            'rules': "",
                            'validator': self.name,
                            'description': self.description,
                            'status': 'ERROR',
                            'message': "Mismatch in checksum on disk and in datapackage.json"
                        })
                    else:
                        status.append({
                            'target': f,
                            'rules': "",
                            'validator': self.name,
                            'description': self.description,
                            'status': 'OK',
                            'message': ""
                        })
                elif f in resource_files:
                    status.append({
                        'target': f,
                        'rules': "",
                        'validator': self.name,
                        'description': self.description,
                        'status': 'ERROR',
                        'message': "In datapackage.json but not in repo"
                    })
                else:
                    status.append({
                        'target': f,
                        'rules': "",
                        'validator': self.name,
                        'description': self.description,
                        'status': 'ERROR',
                        'message': "In repo but not in datapackage.json"
                        })

        return status

Check the integrity of the datapackage . json


def read_file(self, filename):
        """
        Guess the filetype and read the file into row sets
        """
        #print("Reading file", filename)

        try:
            fh = open(filename, 'rb')
            table_set = any_tableset(fh) # guess the type...
        except:
            #traceback.print_exc()
            # Cannot find the schema.
            table_set = None

        return table_set

def
read_file
(
self
,
filename
)
:
#print("Reading file", filename)
try
:
fh
=
open
(
filename
,
'rb'
)
table_set
=
any_tableset
(
fh
)
# guess the type...
except
:
#traceback.print_exc()
# Cannot find the schema.
table_set
=
return
table_set


def get_schema(self, filename):
        """
        Guess schema using messytables
        """
        table_set = self.read_file(filename)

        # Have I been able to read the filename
        if table_set is None:
            return []

        # Get the first table as rowset
        row_set = table_set.tables[0]

        offset, headers = headers_guess(row_set.sample)
        row_set.register_processor(headers_processor(headers))
        row_set.register_processor(offset_processor(offset + 1))
        types = type_guess(row_set.sample, strict=True)

        # Get a sample as well..
        sample = next(row_set.sample)

        clean = lambda v: str(v) if not isinstance(v, str) else v
        schema = []
        for i, h in enumerate(headers):
            schema.append([h,
                           str(types[i]),
                           clean(sample[i].value)])

        return schema

def
get_schema
(
self
,
filename
)
:
table_set
=
self
.
read_file
(
filename
)
# Have I been able to read the filename
if
table_set
is
:
return
[
]
# Get the first table as rowset
row_set
=
table_set
.
tables
[
0
]
offset
,
headers
=
headers_guess
(
row_set
.
sample
)
row_set
.
register_processor
(
headers_processor
(
headers
)
)
row_set
.
register_processor
(
offset_processor
(
offset
+
1
)
)
types
=
type_guess
(
row_set
.
sample
,
strict
=
True
)
# Get a sample as well..
sample
=
next
(
row_set
.
sample
)
clean
=
lambda
v
:
str
(
v
)
if
not
isinstance
(
v
,
str
)
else
v
schema
=
[
]
for
i
,
h
in
enumerate
(
headers
)
:
schema
.
append
(
[
h
,
str
(
types
[
i
]
)
,
clean
(
sample
[
i
]
.
value
)
]
)
return
schema


def int2fin_reference(n):
    """Calculates a checksum for a Finnish national reference number"""
    checksum = 10 - (sum([int(c) * i for c, i in zip(str(n)[::-1], it.cycle((7, 3, 1)))]) % 10)
    if checksum == 10:
        checksum = 0
    return "%s%s" % (n, checksum)

def int2fin_reference ( n ) : checksum = 10 - ( sum ( [ int ( c ) * i for c , i in zip ( str ( n ) [ : : - 1 ] , it . cycle ( ( 7 , 3 , 1 ) ) ) ] ) % 10 ) if checksum == 10 : checksum = 0 return "%s%s" % ( n , checksum )


def iso_reference_valid_char(c, raise_error=True):
    """Helper to make sure the given character is valid for a reference number"""
    if c in ISO_REFERENCE_VALID:
        return True
    if raise_error:
        raise ValueError("'%s' is not in '%s'" % (c, ISO_REFERENCE_VALID))
    return False

def
iso_reference_valid_char
(
c
,
raise_error
=
True
)
:
if
c
in
ISO_REFERENCE_VALID
:
return
True
if
raise_error
:
raise
ValueError
(
"'%s' is not in '%s'"
%
(
c
,
ISO_REFERENCE_VALID
)
)
return
False


def iso_reference_str2int(n):
    """Creates the huge number from ISO alphanumeric ISO reference"""
    n = n.upper()
    numbers = []
    for c in n:
        iso_reference_valid_char(c)
        if c in ISO_REFERENCE_VALID_NUMERIC:
            numbers.append(c)
        else:
            numbers.append(str(iso_reference_char2int(c)))
    return int(''.join(numbers))

def iso_reference_str2int ( n ) : n = n . upper ( ) numbers = [ ] for c in n : iso_reference_valid_char ( c ) if c in ISO_REFERENCE_VALID_NUMERIC : numbers . append ( c ) else : numbers . append ( str ( iso_reference_char2int ( c ) ) ) return int ( '' . join ( numbers ) )


def iso_reference_isvalid(ref):
    """Validates ISO reference number"""
    ref = str(ref)
    cs_source = ref[4:] + ref[:4]
    return (iso_reference_str2int(cs_source) % 97) == 1

def iso_reference_isvalid ( ref ) : ref = str ( ref ) cs_source = ref [ 4 : ] + ref [ : 4 ] return ( iso_reference_str2int ( cs_source ) % 97 ) == 1


def barcode(iban, reference, amount, due=None):
    """Calculates virtual barcode for IBAN account number and ISO reference

    Arguments:
        iban {string} -- IBAN formed account number
        reference {string} -- ISO 11649 creditor reference
        amount {decimal.Decimal} -- Amount in euros, 0.01 - 999999.99
        due {datetime.date} -- due date
    """

    iban = iban.replace(' ', '')
    reference = reference.replace(' ', '')

    if reference.startswith('RF'):
        version = 5
    else:
        version = 4

    if version == 5:
        reference = reference[2:]  # test RF and add 00 where needed
        if len(reference) < 23:
            reference = reference[:2] + ("0" * (23 - len(reference))) + reference[2:]
    elif version == 4:
        reference = reference.zfill(20)

    if not iban.startswith('FI'):
        raise BarcodeException('Barcodes can be printed only for IBANs starting with FI')

    iban = iban[2:]
    amount = "%08d" % (amount.quantize(Decimal('.01')).shift(2).to_integral_value())
    if len(amount) != 8:
        raise BarcodeException("Barcode payment amount must be less than 1000000.00")

    if due:
        due = due.strftime("%y%m%d")
    else:
        due = "000000"

    if version == 4:
        barcode = "%s%s%s000%s%s" % (version, iban, amount, reference, due)
    elif version == 5:
        barcode = "%s%s%s%s%s" % (version, iban, amount, reference, due)

    return barcode


def add_file_normal(f, targetdir, generator,script, source):
    """
    Add a normal file including its source
    """

    basename = os.path.basename(f)
    if targetdir != ".":
        relativepath = os.path.join(targetdir, basename)
    else:
        relativepath = basename

    relpath = os.path.relpath(f, os.getcwd())
    filetype = 'data'
    if script:
        filetype = 'script'
        if generator:
            filetype = 'generator'

    update = OrderedDict([
        ('type', filetype),
        ('generator', generator),
        ('relativepath', relativepath),
        ('content', ""),
        ('source', source),
        ('localfullpath', f),
        ('localrelativepath', relpath)
    ])

    update = annotate_record(update)

    return (basename, update)

def add_file_normal ( f , targetdir , generator , script , source ) : basename = os . path . basename ( f ) if targetdir != "." : relativepath = os . path . join ( targetdir , basename ) else : relativepath = basename relpath = os . path . relpath ( f , os . getcwd ( ) ) filetype = 'data' if script : filetype = 'script' if generator : filetype = 'generator' update = OrderedDict ( [ ( 'type' , filetype ) , ( 'generator' , generator ) , ( 'relativepath' , relativepath ) , ( 'content' , "" ) , ( 'source' , source ) , ( 'localfullpath' , f ) , ( 'localrelativepath' , relpath ) ] ) update = annotate_record ( update ) return ( basename , update )


def run_executable(repo, args, includes):
    """
    Run the executable and capture the input and output...
    """

    # Get platform information
    mgr = plugins_get_mgr()
    repomgr = mgr.get(what='instrumentation', name='platform')
    platform_metadata = repomgr.get_metadata()

    print("Obtaining Commit Information")
    (executable, commiturl) = \
            find_executable_commitpath(repo, args)

    # Create a local directory
    tmpdir = tempfile.mkdtemp()

    # Construct the strace command
    print("Running the command")
    strace_filename = os.path.join(tmpdir,'strace.out.txt')
    cmd = ["strace.py", "-f", "-o", strace_filename,
           "-s", "1024", "-q", "--"] + args

    # Run the command
    p = subprocess.Popen(cmd,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    out, err = p.communicate()

    # Capture the stdout/stderr
    stdout = os.path.join(tmpdir, 'stdout.log.txt')
    with open(stdout, 'w') as fd:
        fd.write(out.decode('utf-8'))

    stderr = os.path.join(tmpdir, 'stderr.log.txt')
    with open(stderr, 'w') as fd:
        fd.write(err.decode('utf-8'))

    # Check the strace output
    files = extract_files(strace_filename, includes)

    # Now insert the execution metadata
    execution_metadata = {
        'likelyexecutable': executable,
        'commitpath': commiturl,
        'args': args,
    }
    execution_metadata.update(platform_metadata)

    for i in range(len(files)):
        files[i]['execution_metadata'] = execution_metadata

    return files

def
run_executable
(
repo
,
args
,
includes
)
:
# Get platform information
mgr
=
plugins_get_mgr
(
)
repomgr
=
mgr
.
get
(
what
=
'instrumentation'
,
name
=
'platform'
)
platform_metadata
=
repomgr
.
get_metadata
(
)
print
(
"Obtaining Commit Information"
)
(
executable
,
commiturl
)
=
find_executable_commitpath
(
repo
,
args
)
# Create a local directory
tmpdir
=
tempfile
.
mkdtemp
(
)
# Construct the strace command
print
(
"Running the command"
)
strace_filename
=
os
.
path
.
join
(
tmpdir
,
'strace.out.txt'
)
cmd
=
[
"strace.py"
,
"-f"
,
"-o"
,
strace_filename
,
"-s"
,
"1024"
,
"-q"
,
"--"
]
+
args
# Run the command
p
=
subprocess
.
Popen
(
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
)
out
,
err
=
p
.
communicate
(
)
# Capture the stdout/stderr
stdout
=
os
.
path
.
join
(
tmpdir
,
'stdout.log.txt'
)
with
open
(
stdout
,
'w'
)
as
fd
:
fd
.
write
(
out
.
decode
(
'utf-8'
)
)
stderr
=
os
.
path
.
join
(
tmpdir
,
'stderr.log.txt'
)
with
open
(
stderr
,
'w'
)
as
fd
:
fd
.
write
(
err
.
decode
(
'utf-8'
)
)
# Check the strace output
files
=
extract_files
(
strace_filename
,
includes
)
# Now insert the execution metadata
execution_metadata
=
{
'likelyexecutable'
:
executable
,
'commitpath'
:
commiturl
,
'args'
:
args
,
}
execution_metadata
.
update
(
platform_metadata
)
for
i
in
range
(
len
(
files
)
)
:
files
[
i
]
[
'execution_metadata'
]
=
execution_metadata
return
files

Run the executable and capture the input and output ...


def add(repo, args, targetdir,
        execute=False, generator=False,
        includes=[], script=False,
        source=None):
    """
    Add files to the repository by explicitly specifying them or by
    specifying a pattern over files accessed during execution of an
    executable.

    Parameters
    ----------

    repo: Repository

    args: files or command line
         (a) If simply adding files, then the list of files that must
         be added (including any additional arguments to be passed to
         git
         (b) If files to be added are an output of a command line, then
         args is the command lined
    targetdir: Target directory to store the files
    execute: Args are not files to be added but scripts that must be run.
    includes: patterns used to select files to
    script: Is this a script?
    generator: Is this a generator
    source: Link to the original source of the data

    """

    # Gather the files...
    if not execute:
        files = add_files(args=args,
                          targetdir=targetdir,
                          source=source,
                          script=script,
                          generator=generator)
    else:
        files = run_executable(repo, args, includes)

    if files is None or len(files) == 0:
        return repo

    # Update the repo package but with only those that have changed.

    filtered_files = []
    package = repo.package
    for h in files:
        found = False
        for i, r in  enumerate(package['resources']):
            if h['relativepath'] == r['relativepath']:
                found = True
                if h['sha256'] == r['sha256']:
                    change = False
                    for attr in ['source']:
                        if h[attr] != r[attr]:
                            r[attr] = h[attr]
                            change = True
                    if change:
                        filtered_files.append(h)
                    continue
                else:
                    filtered_files.append(h)
                    package['resources'][i] = h
                break
        if not found:
            filtered_files.append(h)
            package['resources'].append(h)

    if len(filtered_files) == 0:
        return 0

    # Copy the files
    repo.manager.add_files(repo, filtered_files)

    # Write to disk...
    rootdir = repo.rootdir
    with cd(rootdir):
        datapath = "datapackage.json"
        with open(datapath, 'w') as fd:
            fd.write(json.dumps(package, indent=4))

    return len(filtered_files)

Add files to the repository by explicitly specifying them or by specifying a pattern over files accessed during execution of an executable .


def find_matching_files(self, includes):
        """
        For various actions we need files that match patterns
        """

        if len(includes) == 0:
            return []

        files = [f['relativepath'] for f in self.package['resources']]
        includes = r'|'.join([fnmatch.translate(x) for x in includes])

        # Match both the file name as well the path..
        files = [f for f in files if re.match(includes, os.path.basename(f))] + \
                [f for f in files if re.match(includes, f)]
        files = list(set(files))

        return files


def run(self, cmd, *args):
        """
        Run a specific command using the manager
        """
        if self.manager is None:
            raise Exception("Fatal internal error: Missing repository manager")
        if cmd not in dir(self.manager):
            raise Exception("Fatal internal error: Invalid command {} being run".format(cmd))
        func = getattr(self.manager, cmd)
        repo = self
        return func(repo, *args)


def get_resource(self, p):
        """
        Get metadata for a given file
        """
        for r in self.package['resources']:
            if r['relativepath'] == p:
                r['localfullpath'] = os.path.join(self.rootdir, p)
                return r

        raise Exception("Invalid path")

def
get_resource
(
self
,
p
)
:
for
r
in
self
.
package
[
'resources'
]
:
if
r
[
'relativepath'
]
==
p
:
r
[
'localfullpath'
]
=
os
.
path
.
join
(
self
.
rootdir
,
p
)
return
r
raise
Exception
(
"Invalid path"
)


def lookup(self, username=None, reponame=None, key=None):
        """
        Lookup all available repos
        """
        if key is None:
            key = self.key(username, reponame)
        if key not in self.repos:
            raise UnknownRepository()

        return self.repos[key]

def lookup ( self , username = None , reponame = None , key = None ) : if key is None : key = self . key ( username , reponame ) if key not in self . repos : raise UnknownRepository ( ) return self . repos [ key ]


def rootdir(self,  username, reponame, create=True):
        """
        Working directory for the repo
        """
        path = os.path.join(self.workspace,
                            'datasets',
                            username,
                            reponame)
        if create:
            try:
                os.makedirs(path)
            except:
                pass

        return path

def rootdir ( self , username , reponame , create = True ) : path = os . path . join ( self . workspace , 'datasets' , username , reponame ) if create : try : os . makedirs ( path ) except : pass return path


def add(self, repo):
        """
        Add repo to the internal lookup table...
        """
        key = self.key(repo.username, repo.reponame)
        repo.key = key
        self.repos[key] = repo
        return key

def add ( self , repo ) : key = self . key ( repo . username , repo . reponame ) repo . key = key self . repos [ key ] = repo return key

Add repo to the internal lookup table ...


def lookup(username, reponame):
    """
    Lookup a repo based on username reponame
    """

    mgr = plugins_get_mgr()

    # XXX This should be generalized to all repo managers.
    repomgr = mgr.get(what='repomanager', name='git')
    repo =  repomgr.lookup(username=username,
                           reponame=reponame)
    return repo


def shellcmd(repo, args):
    """
    Run a shell command within the repo's context

    Parameters
    ----------

    repo: Repository object
    args: Shell command
    """
    with cd(repo.rootdir):
        result = run(args)
        return result

def shellcmd ( repo , args ) : with cd ( repo . rootdir ) : result = run ( args ) return result

Run a shell command within the repo s context


def datapackage_exists(repo):
    """
    Check if the datapackage exists...
    """
    datapath = os.path.join(repo.rootdir, "datapackage.json")
    return os.path.exists(datapath)

def datapackage_exists ( repo ) : datapath = os . path . join ( repo . rootdir , "datapackage.json" ) return os . path . exists ( datapath )


def bootstrap_datapackage(repo, force=False,
                          options=None, noinput=False):
    """
    Create the datapackage file..
    """

    print("Bootstrapping datapackage")

    # get the directory
    tsprefix = datetime.now().date().isoformat()

    # Initial data package json
    package = OrderedDict([
        ('title', ''),
        ('description', ''),
        ('username', repo.username),
        ('reponame', repo.reponame),
        ('name', str(repo)),
        ('title', ""),
        ('description', ""),
        ('keywords', []),
        ('resources', []),
        ('creator', getpass.getuser()),
        ('createdat', datetime.now().isoformat()),
        ('remote-url', repo.remoteurl)
    ])

    if options is not None:
        package['title'] = options['title']
        package['description'] = options['description']
    else:
        if noinput:
            raise IncompleteParameters("Option field with title and description")

        for var in ['title', 'description']:
            value = ''
            while value in ['',None]:
                value = input('Your Repo ' + var.title() + ": ")
                if len(value) == 0:
                    print("{} cannot be empty. Please re-enter.".format(var.title()))

            package[var] = value

    # Now store the package...
    (handle, filename) = tempfile.mkstemp()
    with open(filename, 'w') as fd:
        fd.write(json.dumps(package, indent=4))

    repo.package = package

    return filename


def init(username, reponame, setup,
         force=False, options=None,
         noinput=False):
    """
    Initialize an empty repository with datapackage.json

    Parameters
    ----------

    username: Name of the user
    reponame: Name of the repo
    setup: Specify the 'configuration' (git only, git+s3 backend etc)
    force: Force creation of the files
    options: Dictionary with content of dgit.json, if available.
    noinput: Automatic operation with no human interaction
    """

    mgr = plugins_get_mgr()
    repomgr = mgr.get(what='repomanager', name='git')

    backendmgr = None
    if setup == 'git+s3':
        backendmgr = mgr.get(what='backend', name='s3')

    repo = repomgr.init(username, reponame, force, backendmgr)

    # Now bootstrap the datapackage.json metadata file and copy it in...

    # Insert a gitignore with .dgit directory in the repo. This
    # directory will be used to store partial results
    (handle, gitignore) = tempfile.mkstemp()
    with open(gitignore, 'w') as fd:
        fd.write(".dgit")

    # Try to bootstrap. If you cant, cleanup and return
    try:
        filename = bootstrap_datapackage(repo, force, options, noinput)
    except Exception as e:
        repomgr.drop(repo,[])
        os.unlink(gitignore)
        raise e

    repo.run('add_files',
             [
                 {
                     'relativepath': 'datapackage.json',
                     'localfullpath': filename,
                 },
                 {
                     'relativepath': '.gitignore',
                     'localfullpath': gitignore,
                 },
             ])

    # Cleanup temp files
    os.unlink(filename)
    os.unlink(gitignore)

    args = ['-a', '-m', 'Bootstrapped the repo']
    repo.run('commit', args)

    return repo

Initialize an empty repository with datapackage . json


def annotate_metadata_data(repo, task, patterns=["*"], size=0):
    """
    Update metadata with the content of the files
    """

    mgr = plugins_get_mgr()
    keys = mgr.search('representation')['representation']
    representations = [mgr.get_by_key('representation', k) for k in keys]

    matching_files = repo.find_matching_files(patterns)
    package = repo.package
    rootdir = repo.rootdir
    files = package['resources']
    for f in files:
        relativepath = f['relativepath']
        if relativepath in matching_files:
            path = os.path.join(rootdir, relativepath)
            if task == 'preview':
                print("Adding preview for ", relativepath)
                f['content'] = open(path).read()[:size]
            elif task == 'schema':
                for r in representations:
                    if r.can_process(path):
                        print("Adding schema for ", path)
                        f['schema'] = r.get_schema(path)
                        break

def
annotate_metadata_data
(
repo
,
task
,
patterns
=
[
"*"
]
,
size
=
0
)
:
mgr
=
plugins_get_mgr
(
)
keys
=
mgr
.
search
(
'representation'
)
[
'representation'
]
representations
=
[
mgr
.
get_by_key
(
'representation'
,
k
)
for
k
in
keys
]
matching_files
=
repo
.
find_matching_files
(
patterns
)
package
=
repo
.
package
rootdir
=
repo
.
rootdir
files
=
package
[
'resources'
]
for
f
in
files
:
relativepath
=
f
[
'relativepath'
]
if
relativepath
in
matching_files
:
path
=
os
.
path
.
join
(
rootdir
,
relativepath
)
if
task
==
'preview'
:
print
(
"Adding preview for "
,
relativepath
)
f
[
'content'
]
=
open
(
path
)
.
read
(
)
[
:
size
]
elif
task
==
'schema'
:
for
r
in
representations
:
if
r
.
can_process
(
path
)
:
print
(
"Adding schema for "
,
path
)
f
[
'schema'
]
=
r
.
get_schema
(
path
)
break


def annotate_metadata_code(repo, files):
    """
    Update metadata with the commit information
    """

    package = repo.package
    package['code'] = []
    for p in files:
        matching_files = glob2.glob("**/{}".format(p))
        for f in matching_files:
            absf = os.path.abspath(f)
            print("Add commit data for {}".format(f))
            package['code'].append(OrderedDict([
                ('script', f),
                ('permalink', repo.manager.permalink(repo, absf)),
                ('mimetypes', mimetypes.guess_type(absf)[0]),
                ('sha256', compute_sha256(absf))
            ]))

def
annotate_metadata_code
(
repo
,
files
)
:
package
=
repo
.
package
package
[
'code'
]
=
[
]
for
p
in
files
:
matching_files
=
glob2
.
glob
(
"**/{}"
.
format
(
p
)
)
for
f
in
matching_files
:
absf
=
os
.
path
.
abspath
(
f
)
print
(
"Add commit data for {}"
.
format
(
f
)
)
package
[
'code'
]
.
append
(
OrderedDict
(
[
(
'script'
,
f
)
,
(
'permalink'
,
repo
.
manager
.
permalink
(
repo
,
absf
)
)
,
(
'mimetypes'
,
mimetypes
.
guess_type
(
absf
)
[
0
]
)
,
(
'sha256'
,
compute_sha256
(
absf
)
)
]
)
)


def annotate_metadata_action(repo):
    """
    Update metadata with the action history
    """
    package = repo.package

    print("Including history of actions")
    with cd(repo.rootdir):
        filename = ".dgit/log.json"
        if os.path.exists(filename):
            history = open(filename).readlines()
            actions = []
            for a in history:
                try:
                    a = json.loads(a)
                    for x in ['code']:
                        if x not in a or a[x] == None:
                            a[x] = "..."
                    actions.append(a)
                except:
                    pass
            package['actions'] = actions

def
annotate_metadata_action
(
repo
)
:
package
=
repo
.
package
print
(
"Including history of actions"
)
with
cd
(
repo
.
rootdir
)
:
filename
=
".dgit/log.json"
if
os
.
path
.
exists
(
filename
)
:
history
=
open
(
filename
)
.
readlines
(
)
actions
=
[
]
for
a
in
history
:
try
:
a
=
json
.
loads
(
a
)
for
x
in
[
'code'
]
:
if
x
not
in
a
or
a
[
x
]
==
:
a
[
x
]
=
"..."
actions
.
append
(
a
)
except
:
pass
package
[
'actions'
]
=
actions


def annotate_metadata_platform(repo):
    """
    Update metadata host information
    """

    print("Added platform information")
    package = repo.package
    mgr = plugins_get_mgr()
    repomgr = mgr.get(what='instrumentation', name='platform')
    package['platform'] = repomgr.get_metadata()

def
annotate_metadata_platform
(
repo
)
:
print
(
"Added platform information"
)
package
=
repo
.
package
mgr
=
plugins_get_mgr
(
)
repomgr
=
mgr
.
get
(
what
=
'instrumentation'
,
name
=
'platform'
)
package
[
'platform'
]
=
repomgr
.
get_metadata
(
)


def annotate_metadata_dependencies(repo):
    """
    Collect information from the dependent repo's
    """

    options = repo.options

    if 'dependencies' not in options:
        print("No dependencies")
        return []

    repos = []
    dependent_repos = options['dependencies']
    for d in dependent_repos:
        if "/" not in d:
            print("Invalid dependency specification")
        (username, reponame) = d.split("/")
        try:
            repos.append(repo.manager.lookup(username, reponame))
        except:
            print("Repository does not exist. Please create one", d)

    package = repo.package
    package['dependencies'] = []
    for r in repos:
        package['dependencies'].append({
            'username': r.username,
            'reponame': r.reponame,
            })

Collect information from the dependent repo s


def post(repo, args=[]):
    """
    Post to metadata server

    Parameters
    ----------

    repo: Repository object (result of lookup)
    """

    mgr = plugins_get_mgr()
    keys = mgr.search(what='metadata')
    keys = keys['metadata']

    if len(keys) == 0:
        return

    # Incorporate pipeline information...
    if 'pipeline' in repo.options:
        for name, details in repo.options['pipeline'].items():
            patterns = details['files']
            matching_files = repo.find_matching_files(patterns)
            matching_files.sort()
            details['files'] = matching_files
            for i, f in enumerate(matching_files):
                r = repo.get_resource(f)
                if 'pipeline' not in r:
                    r['pipeline'] = []
                r['pipeline'].append(name + " [Step {}]".format(i))

    if 'metadata-management' in repo.options:

        print("Collecting all the required metadata to post")
        metadata = repo.options['metadata-management']

        # Add data repo history
        if 'include-data-history' in metadata and metadata['include-data-history']:
            repo.package['history'] = get_history(repo.rootdir)

        # Add action history
        if 'include-action-history' in metadata and metadata['include-action-history']:
            annotate_metadata_action(repo)

        # Add data repo history
        if 'include-preview' in metadata:
            annotate_metadata_data(repo,
                                   task='preview',
                                   patterns=metadata['include-preview']['files'],
                                   size=metadata['include-preview']['length'])

        if (('include-schema' in metadata) and metadata['include-schema']):
            annotate_metadata_data(repo,  task='schema')

        if 'include-code-history' in metadata:
            annotate_metadata_code(repo, files=metadata['include-code-history'])

        if 'include-platform' in metadata:
            annotate_metadata_platform(repo)

        if 'include-validation' in metadata:
            annotate_metadata_validation(repo)

        if 'include-dependencies' in metadata:
            annotate_metadata_dependencies(repo)

        history = repo.package.get('history',None)
        if (('include-tab-diffs' in metadata) and
            metadata['include-tab-diffs'] and
            history is not None):
            annotate_metadata_diffs(repo)

        # Insert options as well
        repo.package['config'] = repo.options

    try:
        for k in keys:
            # print("Key", k)
            metadatamgr = mgr.get_by_key('metadata', k)
            url = metadatamgr.url
            o = urlparse(url)
            print("Posting to ", o.netloc)
            response = metadatamgr.post(repo)
            if isinstance(response, str):
                print("Error while posting:", response)
            elif response.status_code in [400]:
                content = response.json()
                print("Error while posting:")
                for k in content:
                    print("   ", k,"- ", ",".join(content[k]))
    except NetworkError as e:
        print("Unable to reach metadata server!")
    except NetworkInvalidConfiguration as e:
        print("Invalid network configuration in the INI file")
        print(e.message)
    except Exception as e:
        print("Could not post. Unknown error")
        print(e)


def plugins_show(what=None, name=None, version=None, details=False):
    """
    Show details of available plugins

    Parameters
    ----------
    what: Class of plugins e.g., backend
    name: Name of the plugin e.g., s3
    version: Version of the plugin
    details: Show details be shown?

    """
    global pluginmgr
    return pluginmgr.show(what, name, version, details)

def plugins_show ( what = None , name = None , version = None , details = False ) : global pluginmgr return pluginmgr . show ( what , name , version , details )


def discover_all_plugins(self):
        """
        Load all plugins from dgit extension
        """
        for v in pkg_resources.iter_entry_points('dgit.plugins'):
            m = v.load()
            m.setup(self)

def discover_all_plugins ( self ) : for v in pkg_resources . iter_entry_points ( 'dgit.plugins' ) : m = v . load ( ) m . setup ( self )


def register(self, what, obj):
        """
        Registering a plugin

        Params
        ------
        what: Nature of the plugin (backend, instrumentation, repo)
        obj: Instance of the plugin
        """
        # print("Registering pattern", name, pattern)
        name = obj.name
        version = obj.version
        enable = obj.enable
        if enable == 'n':
            return

        key = Key(name, version)
        self.plugins[what][key] = obj


def search(self, what, name=None, version=None):
        """
        Search for a plugin
        """
        filtered = {}

        # The search may for a scan (what is None) or
        if what is None:
            whats = list(self.plugins.keys())
        elif what is not None:
            if what not in self.plugins:
                raise Exception("Unknown class of plugins")
            whats = [what]
        for what in whats:
            if what not in filtered:
                filtered[what] = []
            for key in self.plugins[what].keys():
                (k_name, k_version) = key
                if name is not None and k_name != name:
                    continue
                if version is not None and k_version != version:
                    continue
                if self.plugins[what][key].enable == 'n':
                    continue
                filtered[what].append(key)

        # print(filtered)
        return filtered


def instantiate(repo, validator_name=None, filename=None, rulesfiles=None):
    """
    Instantiate the validation specification
    """

    default_validators = repo.options.get('validator', {})

    validators = {}
    if validator_name is not None:
        # Handle the case validator is specified..
        if validator_name in default_validators:
            validators = {
                validator_name : default_validators[validator_name]
            }
        else:
            validators = {
                validator_name : {
                    'files': [],
                    'rules': {},
                    'rules-files': []
                }
            }
    else:
        validators = default_validators

    #=========================================
    # Insert the file names
    #=========================================
    if filename is not None:
        matching_files = repo.find_matching_files([filename])
        if len(matching_files) == 0:
            print("Filename could not be found", filename)
            raise Exception("Invalid filename pattern")
        for v in validators:
            validators[v]['files'] = matching_files
    else:
        # Instantiate the files from the patterns specified
        for v in validators:
            if 'files' not in validators[v]:
                validators[v]['files'] = []
            elif len(validators[v]['files']) > 0:
                matching_files = repo.find_matching_files(validators[v]['files'])
                validators[v]['files'] = matching_files

    #=========================================
    # Insert the rules files..
    #=========================================
    if rulesfiles is not None:
        # Command lines...
        matching_files = repo.find_matching_files([rulesfiles])
        if len(matching_files) == 0:
            print("Could not find matching rules files ({}) for {}".format(rulesfiles,v))
            raise Exception("Invalid rules")
        for v in validators:
            validators[v]['rules-files'] = matching_files
    else:
        # Instantiate the files from the patterns specified
        for v in validators:
            if 'rules-files' not in validators[v]:
                validators[v]['rules-files'] = []
            else:
                rulesfiles = validators[v]['rules-files']
                matching_files = repo.find_matching_files(rulesfiles)
                validators[v]['rules-files'] = matching_files

    return validators


def validate(repo,
             validator_name=None,
             filename=None,
             rulesfiles=None,
             args=[]):
    """
    Validate the content of the files for consistency. Validators can
    look as deeply as needed into the files. dgit treats them all as
    black boxes.

    Parameters
    ----------

    repo: Repository object
    validator_name: Name of validator, if any. If none, then all validators specified in dgit.json will be included.
    filename: Pattern that specifies files that must be processed by the validators selected. If none, then the default specification in dgit.json is used.
    rules: Pattern specifying the files that have rules that validators will use
    show: Print the validation results on the terminal

    Returns
    -------

    status: A list of dictionaries, each with target file processed, rules file applied, status of the validation and any error  message.
    """

    mgr = plugins_get_mgr()

    # Expand the specification. Now we have full file paths
    validator_specs = instantiate(repo, validator_name, filename, rulesfiles)

    # Run the validators with rules files...
    allresults = []
    for v in validator_specs:

        keys = mgr.search(what='validator',name=v)['validator']
        for k in keys:
            validator = mgr.get_by_key('validator', k)
            result = validator.evaluate(repo,
                                        validator_specs[v],
                                        args)
            allresults.extend(result)

    return allresults

Validate the content of the files for consistency . Validators can look as deeply as needed into the files . dgit treats them all as black boxes .


def url_is_valid(self, url):
        """
        Check if a URL exists
        """
        # Check if the file system path exists...
        if url.startswith("file://"):
            url = url.replace("file://","")

        return os.path.exists(url)


def post(self, repo):
        """
        Post to the metadata server

        Parameters
        ----------

        repo
        """

        datapackage = repo.package

        url = self.url
        token = self.token
        headers = {
            'Authorization': 'Token {}'.format(token),
            'Content-Type': 'application/json'
        }

        try:
            r = requests.post(url,
                              data = json.dumps(datapackage),
                              headers=headers)

            return r
        except Exception as e:
            #print(e)
            #traceback.print_exc()
            raise NetworkError()
        return ""

def
post
(
self
,
repo
)
:
datapackage
=
repo
.
package
url
=
self
.
url
token
=
self
.
token
headers
=
{
'Authorization'
:
'Token {}'
.
format
(
token
)
,
'Content-Type'
:
'application/json'
}
try
:
r
=
requests
.
post
(
url
,
data
=
json
.
dumps
(
datapackage
)
,
headers
=
headers
)
return
r
except
Exception
as
e
:
#print(e)
#traceback.print_exc()
raise
NetworkError
(
)
return
""


def get_module_class(class_path):
    """
    imports and returns module class from ``path.to.module.Class``
    argument
    """
    mod_name, cls_name = class_path.rsplit('.', 1)

    try:
        mod = import_module(mod_name)
    except ImportError as ex:
        raise EvoStreamException('Error importing module %s: '
                                 '"%s"' % (mod_name, ex))

    return getattr(mod, cls_name)

def
get_module_class
(
class_path
)
:
mod_name
,
cls_name
=
class_path
.
rsplit
(
'.'
,
1
)
try
:
mod
=
import_module
(
mod_name
)
except
ImportError
as
ex
:
raise
EvoStreamException
(
'Error importing module %s: '
'"%s"'
%
(
mod_name
,
ex
)
)
return
getattr
(
mod
,
cls_name
)

imports and returns module class from path . to . module . Class argument


def find_executable_files():
    """
    Find max 5 executables that are responsible for this repo.
    """
    files = glob.glob("*") + glob.glob("*/*") + glob.glob('*/*/*')
    files = filter(lambda f: os.path.isfile(f), files)
    executable = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
    final = []
    for filename in files:
        if os.path.isfile(filename):
            st = os.stat(filename)
            mode = st.st_mode
            if mode & executable:
                final.append(filename)
                if len(final) > 5:
                    break
    return final

def find_executable_files ( ) : files = glob . glob ( "*" ) + glob . glob ( "*/*" ) + glob . glob ( '*/*/*' ) files = filter ( lambda f : os . path . isfile ( f ) , files ) executable = stat . S_IEXEC | stat . S_IXGRP | stat . S_IXOTH final = [ ] for filename in files : if os . path . isfile ( filename ) : st = os . stat ( filename ) mode = st . st_mode if mode & executable : final . append ( filename ) if len ( final ) > 5 : break return final

Find max 5 executables that are responsible for this repo .


def auto_get_repo(autooptions, debug=False):
    """
    Automatically get repo

    Parameters
    ----------

    autooptions: dgit.json content

    """

    # plugin manager
    pluginmgr = plugins_get_mgr()

    # get the repo manager
    repomgr = pluginmgr.get(what='repomanager', name='git')

    repo = None

    try:
        if debug:
            print("Looking repo")
        repo = repomgr.lookup(username=autooptions['username'],
                              reponame=autooptions['reponame'])
    except:
        # Clone the repo
        try:
            print("Checking and cloning if the dataset exists on backend")
            url = autooptions['remoteurl']
            if debug:
                print("Doesnt exist. trying to clone: {}".format(url))
            common_clone(url)
            repo = repomgr.lookup(username=autooptions['username'],
                                  reponame=autooptions['reponame'])
            if debug:
                print("Cloning successful")
        except:
            # traceback.print_exc()
            yes = input("Repo doesnt exist. Should I create one? [yN]")
            if yes == 'y':
                setup = "git"
                if autooptions['remoteurl'].startswith('s3://'):
                    setup = 'git+s3'
                repo = common_init(username=autooptions['username'],
                                   reponame=autooptions['reponame'],
                                   setup=setup,
                                   force=True,
                                   options=autooptions)

                if debug:
                    print("Successfully inited repo")
            else:
                raise Exception("Cannot load repo")

    repo.options = autooptions

    return repo


def get_files_to_commit(autooptions):
    """
    Look through the local directory to pick up files to check
    """
    workingdir = autooptions['working-directory']
    includes = autooptions['track']['includes']
    excludes = autooptions['track']['excludes']

    # transform glob patterns to regular expressions
    # print("Includes ", includes)
    includes = r'|'.join([fnmatch.translate(x) for x in includes])
    excludes = r'|'.join([fnmatch.translate(x) for x in excludes]) or r'$.'

    matched_files = []
    for root, dirs, files in os.walk(workingdir):

        # print("Looking at ", files)

        # exclude dirs
        # dirs[:] = [os.path.join(root, d) for d in dirs]
        dirs[:] = [d for d in dirs if not re.match(excludes, d)]

        # exclude/include files
        files = [f for f in files if not re.match(excludes, f)]
        #print("Files after excludes", files)
        #print(includes)
        files = [f for f in files if re.match(includes, f)]
        #print("Files after includes", files)
        files = [os.path.join(root, f) for f in files]

        matched_files.extend(files)

    return matched_files


def auto_add(repo, autooptions, files):
    """
    Cleanup the paths and add
    """
    # Get the mappings and keys.
    mapping = { ".": "" }
    if (('import' in autooptions) and
        ('directory-mapping' in autooptions['import'])):
        mapping = autooptions['import']['directory-mapping']

    # Apply the longest prefix first...
    keys = mapping.keys()
    keys = sorted(keys, key=lambda k: len(k), reverse=True)

    count = 0
    params = []
    for f in files:

        # Find the destination
        relativepath = f
        for k in keys:
            v = mapping[k]
            if f.startswith(k + "/"):
                #print("Replacing ", k)
                relativepath = f.replace(k + "/", v)
                break

        # Now add to repository
        count += files_add(repo=repo,
                           args=[f],
                           targetdir=os.path.dirname(relativepath))

    return count

def
auto_add
(
repo
,
autooptions
,
files
)
:
# Get the mappings and keys.
mapping
=
{
"."
:
""
}
if
(
(
'import'
in
autooptions
)
and
(
'directory-mapping'
in
autooptions
[
'import'
]
)
)
:
mapping
=
autooptions
[
'import'
]
[
'directory-mapping'
]
# Apply the longest prefix first...
keys
=
mapping
.
keys
(
)
keys
=
sorted
(
keys
,
key
=
lambda
k
:
len
(
k
)
,
reverse
=
True
)
count
=
0
params
=
[
]
for
f
in
files
:
# Find the destination
relativepath
=
f
for
k
in
keys
:
v
=
mapping
[
k
]
if
f
.
startswith
(
k
+
"/"
)
:
#print("Replacing ", k)
relativepath
=
f
.
replace
(
k
+
"/"
,
v
)
break
# Now add to repository
count
+=
files_add
(
repo
=
repo
,
args
=
[
f
]
,
targetdir
=
os
.
path
.
dirname
(
relativepath
)
)
return
count


def pull_stream(self, uri, **kwargs):
        """
        This will try to pull in a stream from an external source. Once a
        stream has been successfully pulled it is assigned a 'local stream
        name' which can be used to access the stream from the EMS.

        :param uri: The URI of the external stream. Can be RTMP, RTSP or
            unicast/multicast (d) mpegts
        :type uri: str

        :param keepAlive: If keepAlive is set to 1, the server will attempt to
            reestablish connection with a stream source after a connection has
            been lost. The reconnect will be attempted once every second
            (default: 1 true)
        :type keepAlive: int

        :param localStreamName: If provided, the stream will be given this
            name. Otherwise, a fallback techniques used to determine the stream
            name (based on the URI)
        :type localStreamName: str

        :param forceTcp: If 1 and if the stream is RTSP, a TCP connection will
            be forced. Otherwise the transport mechanism will be negotiated
            (UDP or TCP) (default: 1 true)
        :type forceTcp: int

        :param tcUrl: When specified, this value will be used to set the TC URL
            in the initial RTMP connect invoke
        :type tcUrl: str

        :param pageUrl: When specified, this value will be used to set the
            originating web page address in the initial RTMP connect invoke
        :type pageUrl: str

        :param swfUrl: When specified, this value will be used to set the
            originating swf URL in the initial RTMP connect invoke
        :type swfUrl: str

        :param rangeStart: For RTSP and RTMP connections. A value from which
            the playback should start expressed in seconds. There are 2 special
            values: -2 and -1. For more information, please read about
            start/len parameters here:
            http://livedocs.adobe.com/flashmediaserver/3.0/hpdocs/help.html?content=00000185.html
        :type rangeStart: int

        :param rangeEnd: The length in seconds for the playback. -1 is a
            special value. For more information, please read about start/len
            parameters here:
            http://livedocs.adobe.com/flashmediaserver/3.0/hpdocs/help.html?content=00000185.html
        :type rangeEnd: int

        :param ttl: Sets the IP_TTL (time to live) option on the socket
        :type ttl: int

        :param tos: Sets the IP_TOS (Type of Service) option on the socket
        :type tos: int

        :param rtcpDetectionInterval: How much time (in seconds) should the
            server wait for RTCP packets before declaring the RTSP stream as a
            RTCP-less stream
        :type rtcpDetectionInterval: int

        :param emulateUserAgent: When specified, this value will be used as the
            user agent string. It is meaningful only for RTMP
        :type emulateUserAgent: str

        :param isAudio: If 1 and if the stream is RTP, it indicates that the
            currently pulled stream is an audio source. Otherwise the pulled
            source is assumed as a video source
        :type isAudio: int

        :param audioCodecBytes: The audio codec setup of this RTP stream if it
            is audio. Represented as hex format without '0x' or 'h'. For
            example: audioCodecBytes=1190
        :type audioCodecBytes: str

        :param spsBytes: The video SPS bytes of this RTP stream if it is video.
            It should be base 64 encoded.
        :type spsBytes: str

        :param ppsBytes: The video PPS bytes of this RTP stream if it is video.
            It should be base 64 encoded
        :type ppsBytes: str

        :param ssmIp: The source IP from source-specific-multicast. Only usable
            when doing UDP based pull
        :type ssmIp: str

        :param httpProxy: This parameter has two valid values: IP:Port - This
            value combination specifies an RTSP HTTP Proxy from which the RTSP
            stream should be pulled from Self - Specifying "self" as the value
            implies pulling RTSP over HTTP
        :type httpProxy: str

        :link: http://docs.evostream.com/ems_api_definition/pullstream
        """
        return self.protocol.execute('pullStream', uri=uri, **kwargs)

def pull_stream ( self , uri , * * kwargs ) : return self . protocol . execute ( 'pullStream' , uri = uri , * * kwargs )

This will try to pull in a stream from an external source . Once a stream has been successfully pulled it is assigned a local stream name which can be used to access the stream from the EMS .


def record(self, localStreamName, pathToFile, **kwargs):
        """
        Records any inbound stream. The record command allows users to record
        a stream that may not yet exist. When a new stream is brought into
        the server, it is checked against a list of streams to be recorded.

        Streams can be recorded as FLV files, MPEG-TS files or as MP4 files.

        :param localStreamName: The name of the stream to be used as input
            for recording.
        :type localStreamName: str

        :param pathToFile: Specify path and file name to write to.
        :type pathToFile: str

        :param type: `ts`, `mp4` or `flv`
        :type type: str

        :param overwrite: If false, when a file already exists for the stream
            name, a new file will be created with the next appropriate number
            appended. If 1 (true), files with the same name will be
            overwritten.
        :type overwrite: int

        :param keepAlive: If 1 (true), the server will restart recording every
            time the stream becomes available again.
        :type keepAlive: int

        :param chunkLength: If non-zero the record command will start a new
            recording file after ChunkLength seconds have elapsed.
        :type chunkLength: int

        :param waitForIDR: This is used if the recording is being chunked.
            When true, new files will only be created on IDR boundaries.
        :type waitForIDR: int

        :param winQtCompat: Mandates 32bit header fields to ensure
            compatibility with Windows QuickTime.
        :type winQtCompat: int

        :param dateFolderStructure: If set to 1 (true), folders will be
            created with names in `YYYYMMDD` format. Recorded files will be
            placed inside these folders based on the date they were created.
        :type dateFolderStructure: int

        :link: http://docs.evostream.com/ems_api_definition/record
        """
        return self.protocol.execute('record',
                                     localStreamName=localStreamName,
                                     pathToFile=pathToFile, **kwargs)

def record ( self , localStreamName , pathToFile , * * kwargs ) : return self . protocol . execute ( 'record' , localStreamName = localStreamName , pathToFile = pathToFile , * * kwargs )

Records any inbound stream . The record command allows users to record a stream that may not yet exist . When a new stream is brought into the server it is checked against a list of streams to be recorded .


def create_ingest_point(self, privateStreamName, publicStreamName):
        """
        Creates an RTMP ingest point, which mandates that streams pushed into
        the EMS have a target stream name which matches one Ingest Point
        privateStreamName.

        :param privateStreamName: The name that RTMP Target Stream Names must
            match.
        :type privateStreamName: str

        :param publicStreamName: The name that is used to access the stream
            pushed to the privateStreamName. The publicStreamName becomes the
            streams localStreamName.
        :type publicStreamName: str

        :link: http://docs.evostream.com/ems_api_definition/createingestpoint
        """
        return self.protocol.execute('createIngestPoint',
                                     privateStreamName=privateStreamName,
                                     publicStreamName=publicStreamName)

def create_ingest_point ( self , privateStreamName , publicStreamName ) : return self . protocol . execute ( 'createIngestPoint' , privateStreamName = privateStreamName , publicStreamName = publicStreamName )

Creates an RTMP ingest point which mandates that streams pushed into the EMS have a target stream name which matches one Ingest Point privateStreamName .


def instantiate(repo, name=None, filename=None):
    """
    Instantiate the generator and filename specification
    """

    default_transformers = repo.options.get('transformer', {})

    # If a name is specified, then lookup the options from dgit.json
    # if specfied. Otherwise it is initialized to an empty list of
    # files.
    transformers = {}
    if name is not None:
        # Handle the case generator is specified..
        if name in default_transformers:
            transformers = {
                name : default_transformers[name]
            }
        else:
            transformers = {
                name : {
                    'files': [],
                }
            }
    else:
        transformers = default_transformers

    #=========================================
    # Map the filename patterns to list of files
    #=========================================
    # Instantiate the files from the patterns specified
    input_matching_files = None
    if filename is not None:
        input_matching_files = repo.find_matching_files([filename])

    for t in transformers:
        for k in transformers[t]:
            if "files" not in k:
                continue
            if k == "files" and input_matching_files is not None:
                # Use the files specified on the command line..
                transformers[t][k] = input_matching_files
            else:
                # Try to match the specification
                if transformers[t][k] is None or len(transformers[t][k]) == 0:
                    transformers[t][k] = []
                else:
                    matching_files = repo.find_matching_files(transformers[t][k])
                    transformers[t][k] = matching_files

    return transformers


def _run(self, cmd):
        """
        Helper function to run commands

        Parameters
        ----------
        cmd : list
              Arguments to git command
        """

        # This is here in case the .gitconfig is not accessible for
        # some reason.
        environ = os.environ.copy()

        environ['GIT_COMMITTER_NAME'] = self.fullname
        environ['GIT_COMMITTER_EMAIL'] = self.email
        environ['GIT_AUTHOR_NAME'] = self.fullname
        environ['GIT_AUTHOR_EMAIL'] = self.email

        cmd = [pipes.quote(c) for c in cmd]
        cmd = " ".join(['/usr/bin/git'] + cmd)
        cmd += "; exit 0"
        #print("Running cmd", cmd)
        try:
            output = subprocess.check_output(cmd,
                                             stderr=subprocess.STDOUT,
                                             shell=True,
                                             env=environ)
        except subprocess.CalledProcessError as e:
            output = e.output

        output = output.decode('utf-8')
        output = output.strip()
        # print("Output of command", output)
        return output


def _run_generic_command(self, repo, cmd):
        """
        Run a generic command within the repo. Assumes that you are
        in the repo's root directory
        """

        result = None
        with cd(repo.rootdir):
            # Dont use sh. It is not collecting the stdout of all
            # child processes.
            output = self._run(cmd)
            try:
                result = {
                    'cmd': cmd,
                    'status': 'success',
                    'message': output,
                }
            except Exception as e:
                result = {
                    'cmd': cmd,
                    'status': 'error',
                    'message': str(e)
                }

        return result

Run a generic command within the repo . Assumes that you are in the repo s root directory


def init(self, username, reponame, force, backend=None):
        """
        Initialize a Git repo

        Parameters
        ----------

        username, reponame : Repo name is tuple (name, reponame)
        force: force initialization of the repo even if exists
        backend: backend that must be used for this (e.g. s3)
        """
        key = self.key(username, reponame)

        # In local filesystem-based server, add a repo
        server_repodir = self.server_rootdir(username,
                                             reponame,
                                             create=False)

        # Force cleanup if needed
        if os.path.exists(server_repodir) and not force:
            raise RepositoryExists()

        if os.path.exists(server_repodir):
            shutil.rmtree(server_repodir)
        os.makedirs(server_repodir)

        # Initialize the repo
        with cd(server_repodir):
            git.init(".", "--bare")

        if backend is not None:
            backend.init_repo(server_repodir)

        # Now clone the filesystem-based repo
        repodir = self.rootdir(username, reponame, create=False)

        # Prepare it if needed
        if os.path.exists(repodir) and not force:
            raise Exception("Local repo already exists")
        if os.path.exists(repodir):
            shutil.rmtree(repodir)
        os.makedirs(repodir)

        # Now clone...
        with cd(os.path.dirname(repodir)):
            git.clone(server_repodir, '--no-hardlinks')

        url = server_repodir
        if backend is not None:
            url = backend.url(username, reponame)

        repo = Repo(username, reponame)
        repo.manager = self
        repo.remoteurl = url
        repo.rootdir = self.rootdir(username, reponame)

        self.add(repo)
        return repo


def delete(self, repo, args=[]):
        """
        Delete files from the repo
        """

        result = None
        with cd(repo.rootdir):
            try:
                cmd = ['rm'] + list(args)
                result = {
                    'status': 'success',
                    'message': self._run(cmd)
                }
            except Exception as e:
                result = {
                    'status': 'error',
                    'message': str(e)
                }

            # print(result)
            return result

def
delete
(
self
,
repo
,
args
=
[
]
)
:
result
=
with
cd
(
repo
.
rootdir
)
:
try
:
cmd
=
[
'rm'
]
+
list
(
args
)
result
=
{
'status'
:
'success'
,
'message'
:
self
.
_run
(
cmd
)
}
except
Exception
as
e
:
result
=
{
'status'
:
'error'
,
'message'
:
str
(
e
)
}
# print(result)
return
result


def drop(self, repo, args=[]):
        """
        Cleanup the repo
        """

        # Clean up the rootdir
        rootdir = repo.rootdir
        if os.path.exists(rootdir):
            print("Cleaning repo directory: {}".format(rootdir))
            shutil.rmtree(rootdir)

        # Cleanup the local version of the repo (this could be on
        # the server etc.
        server_repodir = self.server_rootdir_from_repo(repo,
                                                       create=False)
        if os.path.exists(server_repodir):
            print("Cleaning data from local git 'server': {}".format(server_repodir))
            shutil.rmtree(server_repodir)

        super(GitRepoManager, self).drop(repo)

        return {
            'status': 'success',
            'message': "successful cleanup"
        }