def http_get_provider(provider,
                      request_url, params, token_secret, token_cookie = None):
    '''Handle HTTP GET requests on an authentication endpoint.

    Authentication flow begins when ``params`` has a ``login`` key with a value
    of ``start``. For instance, ``/auth/twitter?login=start``.

    :param str provider: An provider to obtain a user ID from.
    :param str request_url: The authentication endpoint/callback.
    :param dict params: GET parameters from the query string.
    :param str token_secret: An app secret to encode/decode JSON web tokens.
    :param str token_cookie: The current JSON web token, if available.
    :return: A dict containing any of the following possible keys:

        ``status``: an HTTP status code the server should sent

        ``redirect``: where the client should be directed to continue the flow

        ``set_token_cookie``: contains a JSON web token and should be stored by
        the client and passed in the next call.

        ``provider_user_id``: the user ID from the login provider

        ``provider_user_name``: the user name from the login provider
    '''

    if not validate_provider(provider):
        raise InvalidUsage('Provider not supported')

    klass    = getattr(socialauth.providers, provider.capitalize())
    provider = klass(request_url, params, token_secret, token_cookie)
    if provider.status == 302:
        ret = dict(status = 302, redirect = provider.redirect)
        tc  = getattr(provider, 'set_token_cookie', None)
        if tc is not None:
            ret['set_token_cookie'] = tc

        return ret

    if provider.status == 200 and provider.user_id is not None:
        ret = dict(status = 200, provider_user_id = provider.user_id)
        if provider.user_name is not None:
            ret['provider_user_name'] = provider.user_name

        return ret

    raise InvalidUsage('Invalid request')

def
http_get_provider
(
provider
,
request_url
,
params
,
token_secret
,
token_cookie
=
)
:
if
not
validate_provider
(
provider
)
:
raise
InvalidUsage
(
'Provider not supported'
)
klass
=
getattr
(
socialauth
.
providers
,
provider
.
capitalize
(
)
)
provider
=
klass
(
request_url
,
params
,
token_secret
,
token_cookie
)
if
provider
.
status
==
302
:
ret
=
dict
(
status
=
302
,
redirect
=
provider
.
redirect
)
tc
=
getattr
(
provider
,
'set_token_cookie'
,
)
if
tc
is
not
:
ret
[
'set_token_cookie'
]
=
tc
return
ret
if
provider
.
status
==
200
and
provider
.
user_id
is
not
:
ret
=
dict
(
status
=
200
,
provider_user_id
=
provider
.
user_id
)
if
provider
.
user_name
is
not
:
ret
[
'provider_user_name'
]
=
provider
.
user_name
return
ret
raise
InvalidUsage
(
'Invalid request'
)

Handle HTTP GET requests on an authentication endpoint .


def to_json(self):
        """ Method to call to get a serializable object for json.dump or jsonify based on the target

        :return: dict
        """
        if self.subreference is not None:
            return {
                "source": self.objectId,
                "selector": {
                    "type": "FragmentSelector",
                    "conformsTo": "http://ontology-dts.org/terms/subreference",
                    "value": self.subreference
                }
            }
        else:
            return {"source": self.objectId}

Method to call to get a serializable object for json . dump or jsonify based on the target


def read(self):
        """ Read the contents of the Annotation Resource

        :return: the contents of the resource
        :rtype: str or bytes or flask.response
        """
        if not self.__content__:
            self.__retriever__ = self.__resolver__.resolve(self.uri)
            self.__content__, self.__mimetype__ = self.__retriever__.read(self.uri)
        return self.__content__

def read ( self ) : if not self . __content__ : self . __retriever__ = self . __resolver__ . resolve ( self . uri ) self . __content__ , self . __mimetype__ = self . __retriever__ . read ( self . uri ) return self . __content__


def build_index_and_mapping(triples):
    """index all triples into indexes and return their mappings"""
    ents = bidict()
    rels = bidict()
    ent_id = 0
    rel_id = 0

    collected = []
    for t in triples:
        for e in (t.head, t.tail):
            if e not in ents:
                ents[e] = ent_id
                ent_id += 1
        if t.relation not in rels:
            rels[t.relation] = rel_id
            rel_id += 1
        collected.append(kgedata.TripleIndex(ents[t.head], rels[t.relation], ents[t.tail]))

    return collected, ents, rels

def build_index_and_mapping ( triples ) : ents = bidict ( ) rels = bidict ( ) ent_id = 0 rel_id = 0 collected = [ ] for t in triples : for e in ( t . head , t . tail ) : if e not in ents : ents [ e ] = ent_id ent_id += 1 if t . relation not in rels : rels [ t . relation ] = rel_id rel_id += 1 collected . append ( kgedata . TripleIndex ( ents [ t . head ] , rels [ t . relation ] , ents [ t . tail ] ) ) return collected , ents , rels


def recover_triples_from_mapping(indexes, ents: bidict, rels: bidict):
    """recover triples from mapping."""
    triples = []
    for t in indexes:
        triples.append(kgedata.Triple(ents.inverse[t.head], rels.inverse[t.relation], ents.inverse[t.tail]))
    return triples

def recover_triples_from_mapping ( indexes , ents : bidict , rels : bidict ) : triples = [ ] for t in indexes : triples . append ( kgedata . Triple ( ents . inverse [ t . head ] , rels . inverse [ t . relation ] , ents . inverse [ t . tail ] ) ) return triples


def _transform_triple_numpy(x):
    """Transform triple index into a 1-D numpy array."""
    return np.array([x.head, x.relation, x.tail], dtype=np.int64)

def _transform_triple_numpy ( x ) : return np . array ( [ x . head , x . relation , x . tail ] , dtype = np . int64 )

Transform triple index into a 1 - D numpy array .


def pack_triples_numpy(triples):
    """Packs a list of triple indexes into a 2D numpy array."""
    if len(triples) == 0:
        return np.array([], dtype=np.int64)
    return np.stack(list(map(_transform_triple_numpy, triples)), axis=0)

def pack_triples_numpy ( triples ) : if len ( triples ) == 0 : return np . array ( [ ] , dtype = np . int64 ) return np . stack ( list ( map ( _transform_triple_numpy , triples ) ) , axis = 0 )

Packs a list of triple indexes into a 2D numpy array .


def remove_near_duplicate_relation(triples, threshold=0.97):
    """If entity pairs in a relation is as close as another relations, only keep one relation of such set."""
    logging.debug("remove duplicate")

    _assert_threshold(threshold)

    duplicate_rel_counter = defaultdict(list)
    relations = set()
    for t in triples:
        duplicate_rel_counter[t.relation].append(f"{t.head} {t.tail}")
        relations.add(t.relation)
    relations = list(relations)

    num_triples = len(triples)
    removal_relation_set = set()

    for rel, values in duplicate_rel_counter.items():
        duplicate_rel_counter[rel] = Superminhash(values)
    for i in relations:
        for j in relations:
            if i == j or i in removal_relation_set or j in removal_relation_set: continue
            close_relations = [i]
            if _set_close_to(duplicate_rel_counter[i], duplicate_rel_counter[j], threshold):
                close_relations.append(j)
        if len(close_relations) > 1:
            close_relations.pop(np.random.randint(len(close_relations)))
            removal_relation_set |= set(close_relations)
    logging.info("Removing {} relations: {}".format(len(removal_relation_set), str(removal_relation_set)))

    return list(filterfalse(lambda x: x.relation in removal_relation_set, triples))

def
remove_near_duplicate_relation
(
triples
,
threshold
=
0.97
)
:
logging
.
debug
(
"remove duplicate"
)
_assert_threshold
(
threshold
)
duplicate_rel_counter
=
defaultdict
(
list
)
relations
=
set
(
)
for
t
in
triples
:
duplicate_rel_counter
[
t
.
relation
]
.
append
(
f"{t.head} {t.tail}"
)
relations
.
add
(
t
.
relation
)
relations
=
list
(
relations
)
num_triples
=
len
(
triples
)
removal_relation_set
=
set
(
)
for
rel
,
values
in
duplicate_rel_counter
.
items
(
)
:
duplicate_rel_counter
[
rel
]
=
Superminhash
(
values
)
for
i
in
relations
:
for
j
in
relations
:
if
i
==
j
or
i
in
removal_relation_set
or
j
in
removal_relation_set
:
continue
close_relations
=
[
i
]
if
_set_close_to
(
duplicate_rel_counter
[
i
]
,
duplicate_rel_counter
[
j
]
,
threshold
)
:
close_relations
.
append
(
j
)
if
len
(
close_relations
)
>
1
:
close_relations
.
pop
(
np
.
random
.
randint
(
len
(
close_relations
)
)
)
removal_relation_set
|=
set
(
close_relations
)
logging
.
info
(
"Removing {} relations: {}"
.
format
(
len
(
removal_relation_set
)
,
str
(
removal_relation_set
)
)
)
return
list
(
filterfalse
(
lambda
x
:
x
.
relation
in
removal_relation_set
,
triples
)
)

If entity pairs in a relation is as close as another relations only keep one relation of such set .


def remove_direct_link_triples(train, valid, test):
    """Remove direct links in the training sets."""
    pairs = set()
    merged = valid + test
    for t in merged:
        pairs.add((t.head, t.tail))

    filtered = filterfalse(lambda t: (t.head, t.tail) in pairs or (t.tail, t.head) in pairs, train)
    return list(filtered)

def remove_direct_link_triples ( train , valid , test ) : pairs = set ( ) merged = valid + test for t in merged : pairs . add ( ( t . head , t . tail ) ) filtered = filterfalse ( lambda t : ( t . head , t . tail ) in pairs or ( t . tail , t . head ) in pairs , train ) return list ( filtered )

Remove direct links in the training sets .


def shrink_indexes_in_place(self, triples):
        """Uses a union find to find segment."""

        _ent_roots = self.UnionFind(self._ent_id)
        _rel_roots = self.UnionFind(self._rel_id)

        for t in triples:
            _ent_roots.add(t.head)
            _ent_roots.add(t.tail)
            _rel_roots.add(t.relation)

        for i, t in enumerate(triples):
            h = _ent_roots.find(t.head)
            r = _rel_roots.find(t.relation)
            t = _ent_roots.find(t.tail)
            triples[i] = kgedata.TripleIndex(h, r, t)

        ents = bidict()
        available_ent_idx = 0
        for previous_idx, ent_exist in enumerate(_ent_roots.roots()):
            if not ent_exist:
                self._ents.inverse.pop(previous_idx)
            else:
                ents[self._ents.inverse[previous_idx]] = available_ent_idx
            available_ent_idx += 1
        rels = bidict()
        available_rel_idx = 0
        for previous_idx, rel_exist in enumerate(_rel_roots.roots()):
            if not rel_exist:
                self._rels.inverse.pop(previous_idx)
            else:
                rels[self._rels.inverse[previous_idx]] = available_rel_idx
            available_rel_idx += 1
        self._ents = ents
        self._rels = rels
        self._ent_id = available_ent_idx
        self._rel_id = available_rel_idx

def shrink_indexes_in_place ( self , triples ) : _ent_roots = self . UnionFind ( self . _ent_id ) _rel_roots = self . UnionFind ( self . _rel_id ) for t in triples : _ent_roots . add ( t . head ) _ent_roots . add ( t . tail ) _rel_roots . add ( t . relation ) for i , t in enumerate ( triples ) : h = _ent_roots . find ( t . head ) r = _rel_roots . find ( t . relation ) t = _ent_roots . find ( t . tail ) triples [ i ] = kgedata . TripleIndex ( h , r , t ) ents = bidict ( ) available_ent_idx = 0 for previous_idx , ent_exist in enumerate ( _ent_roots . roots ( ) ) : if not ent_exist : self . _ents . inverse . pop ( previous_idx ) else : ents [ self . _ents . inverse [ previous_idx ] ] = available_ent_idx available_ent_idx += 1 rels = bidict ( ) available_rel_idx = 0 for previous_idx , rel_exist in enumerate ( _rel_roots . roots ( ) ) : if not rel_exist : self . _rels . inverse . pop ( previous_idx ) else : rels [ self . _rels . inverse [ previous_idx ] ] = available_rel_idx available_rel_idx += 1 self . _ents = ents self . _rels = rels self . _ent_id = available_ent_idx self . _rel_id = available_rel_idx


def freeze(self):
        """Create a usable data structure for serializing."""
        data = super(IndexBuilder, self).freeze()
        try:
            # Sphinx >= 1.5 format
            # Due to changes from github.com/sphinx-doc/sphinx/pull/2454
            base_file_names = data['docnames']
        except KeyError:
            # Sphinx < 1.5 format
            base_file_names = data['filenames']

        store = {}
        c = itertools.count()
        for prefix, items in iteritems(data['objects']):
            for name, (index, typeindex, _, shortanchor) in iteritems(items):
                objtype = data['objtypes'][typeindex]
                if objtype.startswith('cpp:'):
                    split =  name.rsplit('::', 1)
                    if len(split) != 2:
                        warnings.warn("What's up with %s?" % str((prefix, name, objtype)))
                        continue
                    prefix, name = split
                    last_prefix = prefix.split('::')[-1]
                else:
                    last_prefix = prefix.split('.')[-1]

                store[next(c)] = {
                    'filename': base_file_names[index],
                    'objtype': objtype,
                    'prefix': prefix,
                    'last_prefix': last_prefix,
                    'name': name,
                    'shortanchor': shortanchor,
                }

        data.update({'store': store})
        return data

Create a usable data structure for serializing .


def log_operation(entities, operation_name, params=None):
    """Logs an operation done on an entity, possibly with other arguments
    """
    if isinstance(entities, (list, tuple)):
        entities = list(entities)
    else:
        entities = [entities]

    p = {'name': operation_name, 'on': entities}
    if params:
        p['params'] = params
    _log(TYPE_CODES.OPERATION, p)

def log_operation ( entities , operation_name , params = None ) : if isinstance ( entities , ( list , tuple ) ) : entities = list ( entities ) else : entities = [ entities ] p = { 'name' : operation_name , 'on' : entities } if params : p [ 'params' ] = params _log ( TYPE_CODES . OPERATION , p )

Logs an operation done on an entity possibly with other arguments


def log_state(entity, state):
    """Logs a new state of an entity
    """
    p = {'on': entity, 'state': state}
    _log(TYPE_CODES.STATE, p)

def log_state ( entity , state ) : p = { 'on' : entity , 'state' : state } _log ( TYPE_CODES . STATE , p )


def log_update(entity, update):
    """Logs an update done on an entity
    """
    p = {'on': entity, 'update': update}
    _log(TYPE_CODES.UPDATE, p)

def log_update ( entity , update ) : p = { 'on' : entity , 'update' : update } _log ( TYPE_CODES . UPDATE , p )


def log_error(error, result):
    """Logs an error
    """
    p = {'error': error, 'result':result}
    _log(TYPE_CODES.ERROR, p)

def log_error ( error , result ) : p = { 'error' : error , 'result' : result } _log ( TYPE_CODES . ERROR , p )


def dict_cursor(func):
    """
    Decorator that provides a dictionary cursor to the calling function

    Adds the cursor as the second argument to the calling functions

    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor(cursor_type=CursorType.DICT) coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side dictionary cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor(_CursorType.DICT)) as c:
            return (yield from func(cls, c, *args, **kwargs))

    return wrapper

def dict_cursor ( func ) : @ wraps ( func ) def wrapper ( cls , * args , * * kwargs ) : with ( yield from cls . get_cursor ( _CursorType . DICT ) ) as c : return ( yield from func ( cls , c , * args , * * kwargs ) ) return wrapper


def cursor(func):
    """
    Decorator that provides a cursor to the calling function

    Adds the cursor as the second argument to the calling functions

    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor() coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor()) as c:
            return (yield from func(cls, c, *args, **kwargs))

    return wrapper

def cursor ( func ) : @ wraps ( func ) def wrapper ( cls , * args , * * kwargs ) : with ( yield from cls . get_cursor ( ) ) as c : return ( yield from func ( cls , c , * args , * * kwargs ) ) return wrapper


def nt_cursor(func):
    """
    Decorator that provides a namedtuple cursor to the calling function

    Adds the cursor as the second argument to the calling functions

    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor(cursor_type=CursorType.NAMEDTUPLE) coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side namedtuple cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor(_CursorType.NAMEDTUPLE)) as c:
            return (yield from func(cls, c, *args, **kwargs))

    return wrapper

def nt_cursor ( func ) : @ wraps ( func ) def wrapper ( cls , * args , * * kwargs ) : with ( yield from cls . get_cursor ( _CursorType . NAMEDTUPLE ) ) as c : return ( yield from func ( cls , c , * args , * * kwargs ) ) return wrapper


def transaction(func):
    """
    Provides a transacted cursor which will run in autocommit=false mode

    For any exception the transaction will be rolled back.
    Requires that the function being decorated is an instance of a class or object
    that yields a cursor from a get_cursor(cursor_type=CursorType.NAMEDTUPLE) coroutine or provides such an object
    as the first argument in its signature

    Yields:
        A client-side transacted named cursor
    """

    @wraps(func)
    def wrapper(cls, *args, **kwargs):
        with (yield from cls.get_cursor(_CursorType.NAMEDTUPLE)) as c:
            try:
                yield from c.execute('BEGIN')
                result = (yield from func(cls, c, *args, **kwargs))
            except Exception:
                yield from c.execute('ROLLBACK')
            else:
                yield from c.execute('COMMIT')
                return result

    return wrapper

def transaction ( func ) : @ wraps ( func ) def wrapper ( cls , * args , * * kwargs ) : with ( yield from cls . get_cursor ( _CursorType . NAMEDTUPLE ) ) as c : try : yield from c . execute ( 'BEGIN' ) result = ( yield from func ( cls , c , * args , * * kwargs ) ) except Exception : yield from c . execute ( 'ROLLBACK' ) else : yield from c . execute ( 'COMMIT' ) return result return wrapper

Provides a transacted cursor which will run in autocommit = false mode


def count(cls, cur, table:str, where_keys: list=None):
        """
        gives the number of records in the table

        Args:
            table: a string indicating the name of the table

        Returns:
            an integer indicating the number of records in the table

        """

        if where_keys:
            where_clause, values = cls._get_where_clause_with_values(where_keys)
            query = cls._count_query_where.format(table, where_clause)
            q, t = query, values
        else:
            query = cls._count_query.format(table)
            q, t = query, ()
        yield from cur.execute(q, t)
        result = yield from cur.fetchone()
        return int(result[0])

def count ( cls , cur , table : str , where_keys : list = None ) : if where_keys : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _count_query_where . format ( table , where_clause ) q , t = query , values else : query = cls . _count_query . format ( table ) q , t = query , ( ) yield from cur . execute ( q , t ) result = yield from cur . fetchone ( ) return int ( result [ 0 ] )


def insert(cls, cur, table: str, values: dict):
        """
        Creates an insert statement with only chosen fields

        Args:
            table: a string indicating the name of the table
            values: a dict of fields and values to be inserted

        Returns:
            A 'Record' object with table columns as properties

        """
        keys = cls._COMMA.join(values.keys())
        value_place_holder = cls._PLACEHOLDER * len(values)
        query = cls._insert_string.format(table, keys, value_place_holder[:-1])
        yield from cur.execute(query, tuple(values.values()))
        return (yield from cur.fetchone())

def insert ( cls , cur , table : str , values : dict ) : keys = cls . _COMMA . join ( values . keys ( ) ) value_place_holder = cls . _PLACEHOLDER * len ( values ) query = cls . _insert_string . format ( table , keys , value_place_holder [ : - 1 ] ) yield from cur . execute ( query , tuple ( values . values ( ) ) ) return ( yield from cur . fetchone ( ) )


def update(cls, cur, table: str, values: dict, where_keys: list) -> tuple:
        """
        Creates an update query with only chosen fields
        Supports only a single field where clause

        Args:
            table: a string indicating the name of the table
            values: a dict of fields and values to be inserted
            where_keys: list of dictionary
            example of where keys: [{'name':('>', 'cip'),'url':('=', 'cip.com'},{'type':{'<=', 'manufacturer'}}]
            where_clause will look like ((name>%s and url=%s) or (type <= %s))
            items within each dictionary get 'AND'-ed and dictionaries themselves get 'OR'-ed

        Returns:
            an integer indicating count of rows deleted

        """
        keys = cls._COMMA.join(values.keys())
        value_place_holder = cls._PLACEHOLDER * len(values)
        where_clause, where_values = cls._get_where_clause_with_values(where_keys)
        query = cls._update_string.format(table, keys, value_place_holder[:-1], where_clause)
        yield from cur.execute(query, (tuple(values.values()) + where_values))
        return (yield from cur.fetchall())

def update ( cls , cur , table : str , values : dict , where_keys : list ) -> tuple : keys = cls . _COMMA . join ( values . keys ( ) ) value_place_holder = cls . _PLACEHOLDER * len ( values ) where_clause , where_values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _update_string . format ( table , keys , value_place_holder [ : - 1 ] , where_clause ) yield from cur . execute ( query , ( tuple ( values . values ( ) ) + where_values ) ) return ( yield from cur . fetchall ( ) )


def delete(cls, cur, table: str, where_keys: list):
        """
        Creates a delete query with where keys
        Supports multiple where clause with and or or both

        Args:
            table: a string indicating the name of the table
            where_keys: list of dictionary
            example of where keys: [{'name':('>', 'cip'),'url':('=', 'cip.com'},{'type':{'<=', 'manufacturer'}}]
            where_clause will look like ((name>%s and url=%s) or (type <= %s))
            items within each dictionary get 'AND'-ed and dictionaries themselves get 'OR'-ed

        Returns:
            an integer indicating count of rows deleted

        """
        where_clause, values = cls._get_where_clause_with_values(where_keys)
        query = cls._delete_query.format(table, where_clause)
        yield from cur.execute(query, values)
        return cur.rowcount

def delete ( cls , cur , table : str , where_keys : list ) : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _delete_query . format ( table , where_clause ) yield from cur . execute ( query , values ) return cur . rowcount


def select(cls, cur, table: str, order_by: str, columns: list=None, where_keys: list=None, limit=100,
               offset=0):
        """
        Creates a select query for selective columns with where keys
        Supports multiple where claus with and or or both

        Args:
            table: a string indicating the name of the table
            order_by: a string indicating column name to order the results on
            columns: list of columns to select from
            where_keys: list of dictionary
            limit: the limit on the number of results
            offset: offset on the results

            example of where keys: [{'name':('>', 'cip'),'url':('=', 'cip.com'},{'type':{'<=', 'manufacturer'}}]
            where_clause will look like ((name>%s and url=%s) or (type <= %s))
            items within each dictionary get 'AND'-ed and across dictionaries get 'OR'-ed

        Returns:
            A list of 'Record' object with table columns as properties

        """
        if columns:
            columns_string = cls._COMMA.join(columns)
            if where_keys:
                where_clause, values = cls._get_where_clause_with_values(where_keys)
                query = cls._select_selective_column_with_condition.format(columns_string, table, where_clause,
                                                                           order_by, limit, offset)
                q, t = query, values
            else:
                query = cls._select_selective_column.format(columns_string, table, order_by, limit, offset)
                q, t = query, ()
        else:
            if where_keys:
                where_clause, values = cls._get_where_clause_with_values(where_keys)
                query = cls._select_all_string_with_condition.format(table, where_clause, order_by, limit, offset)
                q, t = query, values
            else:
                query = cls._select_all_string.format(table, order_by, limit, offset)
                q, t = query, ()

        yield from cur.execute(q, t)
        return (yield from cur.fetchall())

def select ( cls , cur , table : str , order_by : str , columns : list = None , where_keys : list = None , limit = 100 , offset = 0 ) : if columns : columns_string = cls . _COMMA . join ( columns ) if where_keys : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _select_selective_column_with_condition . format ( columns_string , table , where_clause , order_by , limit , offset ) q , t = query , values else : query = cls . _select_selective_column . format ( columns_string , table , order_by , limit , offset ) q , t = query , ( ) else : if where_keys : where_clause , values = cls . _get_where_clause_with_values ( where_keys ) query = cls . _select_all_string_with_condition . format ( table , where_clause , order_by , limit , offset ) q , t = query , values else : query = cls . _select_all_string . format ( table , order_by , limit , offset ) q , t = query , ( ) yield from cur . execute ( q , t ) return ( yield from cur . fetchall ( ) )


def raw_sql(cls, cur, query: str, values: tuple):
        """
        Run a raw sql query

        Args:
            query : query string to execute
            values : tuple of values to be used with the query

        Returns:
            result of query as list of named tuple

        """
        yield from cur.execute(query, values)
        return (yield from cur.fetchall())

def raw_sql ( cls , cur , query : str , values : tuple ) : yield from cur . execute ( query , values ) return ( yield from cur . fetchall ( ) )


def serialize_text(out, text):
    """This method is used to append content of the `text`
    argument to the `out` argument.

    Depending on how many lines in the text, a
    padding can be added to all lines except the first
    one.

    Concatenation result is appended to the `out` argument.
    """
    padding = len(out)
    # we need to add padding to all lines
    # except the first one
    add_padding = padding_adder(padding)
    text = add_padding(text, ignore_first_line=True)

    return out + text

def
serialize_text
(
out
,
text
)
:
padding
=
len
(
out
)
# we need to add padding to all lines
# except the first one
add_padding
=
padding_adder
(
padding
)
text
=
add_padding
(
text
,
ignore_first_line
=
True
)
return
out
+
text

This method is used to append content of the text argument to the out argument .


def format_value(value):
    """This function should return unicode representation of the value
    """
    value_id = id(value)

    if value_id in recursion_breaker.processed:
        return u'<recursion>'

    recursion_breaker.processed.add(value_id)

    try:
        if isinstance(value, six.binary_type):
            # suppose, all byte strings are in unicode
            # don't know if everybody in the world uses anything else?
            return u"'{0}'".format(value.decode('utf-8'))

        elif isinstance(value, six.text_type):
            return u"u'{0}'".format(value)

        elif isinstance(value, (list, tuple)):
            # long lists or lists with multiline items
            # will be shown vertically
            values = list(map(format_value, value))
            result = serialize_list(u'[', values, delimiter=u',') + u']'
            return force_unicode(result)

        elif isinstance(value, dict):
            items = six.iteritems(value)

            # format each key/value pair as a text,
            # calling format_value recursively
            items = (tuple(map(format_value, item))
                     for item in items)

            items = list(items)
            # sort by keys for readability
            items.sort()

            # for each item value
            items = [
                serialize_text(
                    u'{0}: '.format(key),
                    item_value)
                for key, item_value in items]

            # and serialize these pieces as a list, enclosing
            # them into a curve brackets
            result = serialize_list(u'{', items, delimiter=u',') + u'}'
            return force_unicode(result)
        return force_unicode(repr(value))

    finally:
        recursion_breaker.processed.remove(value_id)


def traverse(element, query, deep=False):
    """
    Helper function to traverse an element tree rooted at element, yielding nodes matching the query.
    """
    # Grab the next part of the query (it will be chopped from the front each iteration).
    part = query[0]
    if not part:
        # If the part is blank, we encountered a //, meaning search all sub-nodes.
        query = query[1:]
        part = query[0]
        deep = True
    # Parse out any predicate (tag[pred]) from this part of the query.
    part, predicate = xpath_re.match(query[0]).groups()
    for c in element._children:
        if part in ('*', c.tagname) and c._match(predicate):
            # A potential matching branch: this child matches the next query part (and predicate).
            if len(query) == 1:
                # If this is the last part of the query, we found a matching element, yield it.
                yield c
            else:
                # Otherwise, check the children of this child against the next query part.
                for e in traverse(c, query[1:]):
                    yield e
        if deep:
            # If we're searching all sub-nodes, traverse with the same query, regardless of matching.
            # This basically creates a recursion branch to search EVERYWHERE for anything after //.
            for e in traverse(c, query, deep=True):
                yield e

Helper function to traverse an element tree rooted at element yielding nodes matching the query .


def parse_query(query):
    """
    Given a simplified XPath query string, returns an array of normalized query parts.
    """
    parts = query.split('/')
    norm = []
    for p in parts:
        p = p.strip()
        if p:
            norm.append(p)
        elif '' not in norm:
            norm.append('')
    return norm

def parse_query ( query ) : parts = query . split ( '/' ) norm = [ ] for p in parts : p = p . strip ( ) if p : norm . append ( p ) elif '' not in norm : norm . append ( '' ) return norm

Given a simplified XPath query string returns an array of normalized query parts .


def insert(self, before, name, attrs=None, data=None):
        """
        Inserts a new element as a child of this element, before the specified index or sibling.

        :param before: An :class:`XmlElement` or a numeric index to insert the new node before
        :param name: The tag name to add
        :param attrs: Attributes for the new tag
        :param data: CDATA for the new tag
        :returns: The newly-created element
        :rtype: :class:`XmlElement`
        """
        if isinstance(before, self.__class__):
            if before.parent != self:
                raise ValueError('Cannot insert before an element with a different parent.')
            before = before.index
        # Make sure 0 <= before <= len(_children).
        before = min(max(0, before), len(self._children))
        elem = self.__class__(name, attrs, data, parent=self, index=before)
        self._children.insert(before, elem)
        # Re-index all the children.
        for idx, c in enumerate(self._children):
            c.index = idx
        return elem

Inserts a new element as a child of this element before the specified index or sibling .


def children(self, name=None, reverse=False):
        """
        A generator yielding children of this node.

        :param name: If specified, only consider elements with this tag name
        :param reverse: If ``True``, children will be yielded in reverse declaration order
        """
        elems = self._children
        if reverse:
            elems = reversed(elems)
        for elem in elems:
            if name is None or elem.tagname == name:
                yield elem

def children ( self , name = None , reverse = False ) : elems = self . _children if reverse : elems = reversed ( elems ) for elem in elems : if name is None or elem . tagname == name : yield elem

A generator yielding children of this node .


def _match(self, pred):
        """
        Helper function to determine if this node matches the given predicate.
        """
        if not pred:
            return True
        # Strip off the [ and ]
        pred = pred[1:-1]
        if pred.startswith('@'):
            # An attribute predicate checks the existence (and optionally value) of an attribute on this tag.
            pred = pred[1:]
            if '=' in pred:
                attr, value = pred.split('=', 1)
                if value[0] in ('"', "'"):
                    value = value[1:]
                if value[-1] in ('"', "'"):
                    value = value[:-1]
                return self.attrs.get(attr) == value
            else:
                return pred in self.attrs
        elif num_re.match(pred):
            # An index predicate checks whether we are the n-th child of our parent (0-based).
            index = int(pred)
            if index < 0:
                if self.parent:
                    # For negative indexes, count from the end of the list.
                    return self.index == (len(self.parent._children) + index)
                else:
                    # If we're the root node, the only index we could be is 0.
                    return index == 0
            else:
                return index == self.index
        else:
            if '=' in pred:
                tag, value = pred.split('=', 1)
                if value[0] in ('"', "'"):
                    value = value[1:]
                if value[-1] in ('"', "'"):
                    value = value[:-1]
                for c in self._children:
                    if c.tagname == tag and c.data == value:
                        return True
            else:
                # A plain [tag] predicate means we match if we have a child with tagname "tag".
                for c in self._children:
                    if c.tagname == pred:
                        return True
        return False

Helper function to determine if this node matches the given predicate .


def path(self, include_root=False):
        """
        Returns a canonical path to this element, relative to the root node.

        :param include_root: If ``True``, include the root node in the path. Defaults to ``False``.
        """
        path = '%s[%d]' % (self.tagname, self.index or 0)
        p = self.parent
        while p is not None:
            if p.parent or include_root:
                path = '%s[%d]/%s' % (p.tagname, p.index or 0, path)
            p = p.parent
        return path

def path ( self , include_root = False ) : path = '%s[%d]' % ( self . tagname , self . index or 0 ) p = self . parent while p is not None : if p . parent or include_root : path = '%s[%d]/%s' % ( p . tagname , p . index or 0 , path ) p = p . parent return path

Returns a canonical path to this element relative to the root node .


def iter(self, name=None):
        """
        Recursively find any descendants of this node with the given tag name. If a tag name is omitted, this will
        yield every descendant node.

        :param name: If specified, only consider elements with this tag name
        :returns: A generator yielding descendants of this node
        """
        for c in self._children:
            if name is None or c.tagname == name:
                yield c
            for gc in c.find(name):
                yield gc

def iter ( self , name = None ) : for c in self . _children : if name is None or c . tagname == name : yield c for gc in c . find ( name ) : yield gc

Recursively find any descendants of this node with the given tag name . If a tag name is omitted this will yield every descendant node .


def last(self, name=None):
        """
        Returns the last child of this node.

        :param name: If specified, only consider elements with this tag name
        :rtype: :class:`XmlElement`
        """
        for c in self.children(name, reverse=True):
            return c

def last ( self , name = None ) : for c in self . children ( name , reverse = True ) : return c


def parents(self, name=None):
        """
        Yields all parents of this element, back to the root element.

        :param name: If specified, only consider elements with this tag name
        """
        p = self.parent
        while p is not None:
            if name is None or p.tagname == name:
                yield p
            p = p.parent

def parents ( self , name = None ) : p = self . parent while p is not None : if name is None or p . tagname == name : yield p p = p . parent

Yields all parents of this element back to the root element .


def next(self, name=None):
        """
        Returns the next sibling of this node.

        :param name: If specified, only consider elements with this tag name
        :rtype: :class:`XmlElement`
        """
        if self.parent is None or self.index is None:
            return None
        for idx in xrange(self.index + 1, len(self.parent)):
            if name is None or self.parent[idx].tagname == name:
                return self.parent[idx]

def next ( self , name = None ) : if self . parent is None or self . index is None : return None for idx in xrange ( self . index + 1 , len ( self . parent ) ) : if name is None or self . parent [ idx ] . tagname == name : return self . parent [ idx ]


def prev(self, name=None):
        """
        Returns the previous sibling of this node.

        :param name: If specified, only consider elements with this tag name
        :rtype: :class:`XmlElement`
        """
        if self.parent is None or self.index is None:
            return None
        for idx in xrange(self.index - 1, -1, -1):
            if name is None or self.parent[idx].tagname == name:
                return self.parent[idx]

def prev ( self , name = None ) : if self . parent is None or self . index is None : return None for idx in xrange ( self . index - 1 , - 1 , - 1 ) : if name is None or self . parent [ idx ] . tagname == name : return self . parent [ idx ]

Returns the previous sibling of this node .


def get_observations(self):
        """
        Parses the HTML table into a list of dictionaries, each of which
        represents a single observation.
        """
        if self.empty:
            return []
        rows = list(self.tbody)
        observations = []
        for row_observation, row_details in zip(rows[::2], rows[1::2]):
            data = {}
            cells = OBSERVATION_XPATH(row_observation)
            data['name'] = _clean_cell(cells[0])
            data['date'] = _clean_cell(cells[1])
            data['magnitude'] = _clean_cell(cells[3])
            data['obscode'] = _clean_cell(cells[6])
            cells = DETAILS_XPATH(row_details)
            data['comp1'] = _clean_cell(cells[0])
            data['chart'] = _clean_cell(cells[3]).replace('None', '')
            data['comment_code'] = _clean_cell(cells[4])
            data['notes'] = _clean_cell(cells[5])
            observations.append(data)
        return observations

def get_observations ( self ) : if self . empty : return [ ] rows = list ( self . tbody ) observations = [ ] for row_observation , row_details in zip ( rows [ : : 2 ] , rows [ 1 : : 2 ] ) : data = { } cells = OBSERVATION_XPATH ( row_observation ) data [ 'name' ] = _clean_cell ( cells [ 0 ] ) data [ 'date' ] = _clean_cell ( cells [ 1 ] ) data [ 'magnitude' ] = _clean_cell ( cells [ 3 ] ) data [ 'obscode' ] = _clean_cell ( cells [ 6 ] ) cells = DETAILS_XPATH ( row_details ) data [ 'comp1' ] = _clean_cell ( cells [ 0 ] ) data [ 'chart' ] = _clean_cell ( cells [ 3 ] ) . replace ( 'None' , '' ) data [ 'comment_code' ] = _clean_cell ( cells [ 4 ] ) data [ 'notes' ] = _clean_cell ( cells [ 5 ] ) observations . append ( data ) return observations

Parses the HTML table into a list of dictionaries each of which represents a single observation .


def get_cache_key(prefix, *args, **kwargs):
    """
    Calculates cache key based on `args` and `kwargs`.
    `args` and `kwargs` must be instances of hashable types.
    """
    hash_args_kwargs = hash(tuple(kwargs.iteritems()) + args)
    return '{}_{}'.format(prefix, hash_args_kwargs)

def get_cache_key ( prefix , * args , * * kwargs ) : hash_args_kwargs = hash ( tuple ( kwargs . iteritems ( ) ) + args ) return '{}_{}' . format ( prefix , hash_args_kwargs )

Calculates cache key based on args and kwargs . args and kwargs must be instances of hashable types .


def cache_func(prefix, method=False):
    """
    Cache result of function execution into the django cache backend.
    Calculate cache key based on `prefix`, `args` and `kwargs` of the function.
    For using like object method set `method=True`.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            cache_args = args
            if method:
                cache_args = args[1:]
            cache_key = get_cache_key(prefix, *cache_args, **kwargs)
            cached_value = cache.get(cache_key)
            if cached_value is None:
                cached_value = func(*args, **kwargs)
                cache.set(cache_key, cached_value)
            return cached_value
        return wrapper
    return decorator

def cache_func ( prefix , method = False ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , * * kwargs ) : cache_args = args if method : cache_args = args [ 1 : ] cache_key = get_cache_key ( prefix , * cache_args , * * kwargs ) cached_value = cache . get ( cache_key ) if cached_value is None : cached_value = func ( * args , * * kwargs ) cache . set ( cache_key , cached_value ) return cached_value return wrapper return decorator

Cache result of function execution into the django cache backend . Calculate cache key based on prefix args and kwargs of the function . For using like object method set method = True .


def get_or_default(func=None, default=None):
    """
    Wrapper around Django's ORM `get` functionality.
    Wrap anything that raises ObjectDoesNotExist exception
    and provide the default value if necessary.
    `default` by default is None. `default` can be any callable,
    if it is callable it will be called when ObjectDoesNotExist
    exception will be raised.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except ObjectDoesNotExist:
                if callable(default):
                    return default()
                else:
                    return default
        return wrapper
    if func is None:
        return decorator
    else:
        return decorator(func)

def get_or_default ( func = None , default = None ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , * * kwargs ) : try : return func ( * args , * * kwargs ) except ObjectDoesNotExist : if callable ( default ) : return default ( ) else : return default return wrapper if func is None : return decorator else : return decorator ( func )

Wrapper around Django s ORM get functionality . Wrap anything that raises ObjectDoesNotExist exception and provide the default value if necessary . default by default is None . default can be any callable if it is callable it will be called when ObjectDoesNotExist exception will be raised .


def _get_column_nums_from_args(columns):
    """Turn column inputs from user into list of simple numbers.

    Inputs can be:

      - individual number: 1
      - range: 1-3
      - comma separated list: 1,2,3,4-6
    """
    nums = []
    for c in columns:
        for p in c.split(','):
            p = p.strip()
            try:
                c = int(p)
                nums.append(c)
            except (TypeError, ValueError):
                start, ignore, end = p.partition('-')
                try:
                    start = int(start)
                    end = int(end)
                except (TypeError, ValueError):
                    raise ValueError(
                        'Did not understand %r, expected digit-digit' % c
                    )
                inc = 1 if start < end else -1
                nums.extend(range(start, end + inc, inc))
    # The user will pass us 1-based indexes, but we need to use
    # 0-based indexing with the row.
    return [n - 1 for n in nums]

Turn column inputs from user into list of simple numbers .


def _get_printable_columns(columns, row):
    """Return only the part of the row which should be printed.
    """
    if not columns:
        return row

    # Extract the column values, in the order specified.
    return tuple(row[c] for c in columns)

Return only the part of the row which should be printed .


def writerow(self, observation_data):
        """
        Writes a single observation to the output file.

        If the ``observation_data`` parameter is a dictionary, it is
        converted to a list to keep a consisted field order (as described
        in format specification). Otherwise it is assumed that the data
        is a raw record ready to be written to file.

        :param observation_data: a single observation as a dictionary or list
        """
        if isinstance(observation_data, (list, tuple)):
            row = observation_data
        else:
            row = self.dict_to_row(observation_data)
        self.writer.writerow(row)

def writerow ( self , observation_data ) : if isinstance ( observation_data , ( list , tuple ) ) : row = observation_data else : row = self . dict_to_row ( observation_data ) self . writer . writerow ( row )

Writes a single observation to the output file .


def dict_to_row(cls, observation_data):
        """
        Takes a dictionary of observation data and converts it to a list
        of fields according to AAVSO visual format specification.

        :param cls: current class
        :param observation_data: a single observation as a dictionary
        """
        row = []
        row.append(observation_data['name'])
        row.append(observation_data['date'])
        row.append(observation_data['magnitude'])
        comment_code = observation_data.get('comment_code', 'na')
        if not comment_code:
            comment_code = 'na'
        row.append(comment_code)
        comp1 = observation_data.get('comp1', 'na')
        if not comp1:
            comp1 = 'na'
        row.append(comp1)
        comp2 = observation_data.get('comp2', 'na')
        if not comp2:
            comp2 = 'na'
        row.append(comp2)
        chart = observation_data.get('chart', 'na')
        if not chart:
            chart = 'na'
        row.append(chart)
        notes = observation_data.get('notes', 'na')
        if not notes:
            notes = 'na'
        row.append(notes)
        return row

def dict_to_row ( cls , observation_data ) : row = [ ] row . append ( observation_data [ 'name' ] ) row . append ( observation_data [ 'date' ] ) row . append ( observation_data [ 'magnitude' ] ) comment_code = observation_data . get ( 'comment_code' , 'na' ) if not comment_code : comment_code = 'na' row . append ( comment_code ) comp1 = observation_data . get ( 'comp1' , 'na' ) if not comp1 : comp1 = 'na' row . append ( comp1 ) comp2 = observation_data . get ( 'comp2' , 'na' ) if not comp2 : comp2 = 'na' row . append ( comp2 ) chart = observation_data . get ( 'chart' , 'na' ) if not chart : chart = 'na' row . append ( chart ) notes = observation_data . get ( 'notes' , 'na' ) if not notes : notes = 'na' row . append ( notes ) return row

Takes a dictionary of observation data and converts it to a list of fields according to AAVSO visual format specification .


def row_to_dict(cls, row):
        """
        Converts a raw input record to a dictionary of observation data.

        :param cls: current class
        :param row: a single observation as a list or tuple
        """
        comment_code = row[3]
        if comment_code.lower() == 'na':
            comment_code = ''
        comp1 = row[4]
        if comp1.lower() == 'na':
            comp1 = ''
        comp2 = row[5]
        if comp2.lower() == 'na':
            comp2 = ''
        chart = row[6]
        if chart.lower() == 'na':
            chart = ''
        notes = row[7]
        if notes.lower() == 'na':
            notes = ''
        return {
            'name': row[0],
            'date': row[1],
            'magnitude': row[2],
            'comment_code': comment_code,
            'comp1': comp1,
            'comp2': comp2,
            'chart': chart,
            'notes': notes,
        }

def row_to_dict ( cls , row ) : comment_code = row [ 3 ] if comment_code . lower ( ) == 'na' : comment_code = '' comp1 = row [ 4 ] if comp1 . lower ( ) == 'na' : comp1 = '' comp2 = row [ 5 ] if comp2 . lower ( ) == 'na' : comp2 = '' chart = row [ 6 ] if chart . lower ( ) == 'na' : chart = '' notes = row [ 7 ] if notes . lower ( ) == 'na' : notes = '' return { 'name' : row [ 0 ] , 'date' : row [ 1 ] , 'magnitude' : row [ 2 ] , 'comment_code' : comment_code , 'comp1' : comp1 , 'comp2' : comp2 , 'chart' : chart , 'notes' : notes , }

Converts a raw input record to a dictionary of observation data .


def get_default_tag(app):
    '''Get the name of the view function used to prevent having to set the tag
    manually for every endpoint'''
    view_func = get_view_function(app, request.path, request.method)
    if view_func:
        return view_func.__name__

def get_default_tag ( app ) : view_func = get_view_function ( app , request . path , request . method ) if view_func : return view_func . __name__


def download_observations(observer_code):
    """
    Downloads all variable star observations by a given observer.

    Performs a series of HTTP requests to AAVSO's WebObs search and
    downloads the results page by page. Each page is then passed to
    :py:class:`~pyaavso.parsers.webobs.WebObsResultsParser` and parse results
    are added to the final observation list.
    """
    page_number = 1
    observations = []
    while True:
        logger.info('Downloading page %d...', page_number)
        response = requests.get(WEBOBS_RESULTS_URL, params={
            'obscode': observer_code,
            'num_results': 200,
            'obs_types': 'all',
            'page': page_number,
        })
        logger.debug(response.request.url)
        parser = WebObsResultsParser(response.text)
        observations.extend(parser.get_observations())
        # kinda silly, but there's no need for lxml machinery here
        if '>Next</a>' not in response.text:
            break
        page_number += 1
    return observations

Downloads all variable star observations by a given observer .


def image_path(instance, filename):
    """Generates likely unique image path using md5 hashes"""
    filename, ext = os.path.splitext(filename.lower())
    instance_id_hash = hashlib.md5(str(instance.id)).hexdigest()
    filename_hash = ''.join(random.sample(hashlib.md5(filename.encode('utf-8')).hexdigest(), 8))
    return '{}/{}{}'.format(instance_id_hash, filename_hash, ext)

def image_path ( instance , filename ) : filename , ext = os . path . splitext ( filename . lower ( ) ) instance_id_hash = hashlib . md5 ( str ( instance . id ) ) . hexdigest ( ) filename_hash = '' . join ( random . sample ( hashlib . md5 ( filename . encode ( 'utf-8' ) ) . hexdigest ( ) , 8 ) ) return '{}/{}{}' . format ( instance_id_hash , filename_hash , ext )


async def process_lander_page(session, github_api_token, ltd_product_data,
                              mongo_collection=None):
    """Extract, transform, and load metadata from Lander-based projects.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    ltd_product_data : `dict`
        Contents of ``metadata.yaml``, obtained via `download_metadata_yaml`.
        Data for this technote from the LTD Keeper API
        (``GET /products/<slug>``). Usually obtained via
        `lsstprojectmeta.ltd.get_ltd_product`.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records. If provided, ths JSON-LD is upserted
        into the MongoDB collection.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    Raises
    ------
    NotLanderPageError
        Raised when the LTD product cannot be interpreted as a Lander page
        because the ``/metadata.jsonld`` file is absent. This implies that
        the LTD product *could* be of a different format.

    .. `GitHub personal access token guide`: https://ls.st/41d
    """
    logger = logging.getLogger(__name__)

    # Try to download metadata.jsonld from the Landing page site.
    published_url = ltd_product_data['published_url']
    jsonld_url = urljoin(published_url, '/metadata.jsonld')
    try:
        async with session.get(jsonld_url) as response:
            logger.debug('%s response status %r', jsonld_url, response.status)
            response.raise_for_status()
            json_data = await response.text()
    except aiohttp.ClientResponseError as err:
        logger.debug('Tried to download %s, got status %d',
                     jsonld_url, err.code)
        raise NotLanderPageError()
    # Use our own json parser to get datetimes
    metadata = decode_jsonld(json_data)

    if mongo_collection is not None:
        await _upload_to_mongodb(mongo_collection, metadata)

    return metadata

Extract transform and load metadata from Lander - based projects .


async def _upload_to_mongodb(collection, jsonld):
    """Upsert the technote resource into the projectmeta MongoDB collection.

    Parameters
    ----------
    collection : `motor.motor_asyncio.AsyncIOMotorCollection`
        The MongoDB collection.
    jsonld : `dict`
        The JSON-LD document that represents the document resource.
    """
    document = {
        'data': jsonld
    }
    query = {
        'data.reportNumber': jsonld['reportNumber']
    }
    await collection.update(query, document, upsert=True, multi=False)

async def _upload_to_mongodb ( collection , jsonld ) : document = { 'data' : jsonld } query = { 'data.reportNumber' : jsonld [ 'reportNumber' ] } await collection . update ( query , document , upsert = True , multi = False )

Upsert the technote resource into the projectmeta MongoDB collection .


def json_doc_to_xml(json_obj, lang='en', custom_namespace=None):
    """Converts a Open511 JSON document to XML.

    lang: the appropriate language code

    Takes a dict deserialized from JSON, returns an lxml Element.

    Accepts only the full root-level JSON object from an Open511 response."""
    if 'meta' not in json_obj:
        raise Exception("This function requires a conforming Open511 JSON document with a 'meta' section.")
    json_obj = dict(json_obj)
    meta = json_obj.pop('meta')
    elem = get_base_open511_element(lang=lang, version=meta.pop('version'))

    pagination = json_obj.pop('pagination', None)

    json_struct_to_xml(json_obj, elem, custom_namespace=custom_namespace)

    if pagination:
        elem.append(json_struct_to_xml(pagination, 'pagination', custom_namespace=custom_namespace))

    json_struct_to_xml(meta, elem)

    return elem

Converts a Open511 JSON document to XML .


def json_struct_to_xml(json_obj, root, custom_namespace=None):
    """Converts a Open511 JSON fragment to XML.

    Takes a dict deserialized from JSON, returns an lxml Element.

    This won't provide a conforming document if you pass in a full JSON document;
    it's for translating little fragments, and is mostly used internally."""
    if isinstance(root, (str, unicode)):
        if root.startswith('!'):
            root = etree.Element('{%s}%s' % (NS_PROTECTED, root[1:]))
        elif root.startswith('+'):
            if not custom_namespace:
                raise Exception("JSON fields starts with +, but no custom namespace provided")
            root = etree.Element('{%s}%s' % (custom_namespace, root[1:]))
        else:
            root = etree.Element(root)
    if root.tag in ('attachments', 'grouped_events', 'media_files'):
        for link in json_obj:
            root.append(json_link_to_xml(link))
    elif isinstance(json_obj, (str, unicode)):
        root.text = json_obj
    elif isinstance(json_obj, (int, float)):
        root.text = unicode(json_obj)
    elif isinstance(json_obj, dict):
        if frozenset(json_obj.keys()) == frozenset(('type', 'coordinates')):
            root.append(geojson_to_gml(json_obj))
        else:
            for key, val in json_obj.items():
                if key == 'url' or key.endswith('_url'):
                    el = json_link_to_xml(val, json_link_key_to_xml_rel(key))
                else:
                    el = json_struct_to_xml(val, key, custom_namespace=custom_namespace)
                if el is not None:
                    root.append(el)
    elif isinstance(json_obj, list):
        tag_name = root.tag
        if tag_name.endswith('ies'):
            tag_name = tag_name[:-3] + 'y'
        elif tag_name.endswith('s'):
            tag_name = tag_name[:-1]
        for val in json_obj:
            el = json_struct_to_xml(val, tag_name, custom_namespace=custom_namespace)
            if el is not None:
                root.append(el)
    elif json_obj is None:
        return None
    else:
        raise NotImplementedError
    return root

Converts a Open511 JSON fragment to XML .


def geojson_to_gml(gj, set_srs=True):
    """Given a dict deserialized from a GeoJSON object, returns an lxml Element
    of the corresponding GML geometry."""
    tag = G(gj['type'])
    if set_srs:
        tag.set('srsName', 'urn:ogc:def:crs:EPSG::4326')

    if gj['type'] == 'Point':
        tag.append(G.pos(_reverse_geojson_coords(gj['coordinates'])))
    elif gj['type'] == 'LineString':
        tag.append(G.posList(' '.join(_reverse_geojson_coords(ll) for ll in gj['coordinates'])))
    elif gj['type'] == 'Polygon':
        rings = [
            G.LinearRing(
                G.posList(' '.join(_reverse_geojson_coords(ll) for ll in ring))
            ) for ring in gj['coordinates']
        ]
        tag.append(G.exterior(rings.pop(0)))
        for ring in rings:
            tag.append(G.interior(ring))
    elif gj['type'] in ('MultiPoint', 'MultiLineString', 'MultiPolygon'):
        single_type = gj['type'][5:]
        member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        for coord in gj['coordinates']:
            tag.append(
                G(member_tag, geojson_to_gml({'type': single_type, 'coordinates': coord}, set_srs=False))
            )
    else:
        raise NotImplementedError

    return tag

def
geojson_to_gml
(
gj
,
set_srs
=
True
)
:
tag
=
G
(
gj
[
'type'
]
)
if
set_srs
:
tag
.
set
(
'srsName'
,
'urn:ogc:def:crs:EPSG::4326'
)
if
gj
[
'type'
]
==
'Point'
:
tag
.
append
(
G
.
pos
(
_reverse_geojson_coords
(
gj
[
'coordinates'
]
)
)
)
elif
gj
[
'type'
]
==
'LineString'
:
tag
.
append
(
G
.
posList
(
' '
.
join
(
_reverse_geojson_coords
(
ll
)
for
ll
in
gj
[
'coordinates'
]
)
)
)
elif
gj
[
'type'
]
==
'Polygon'
:
rings
=
[
G
.
LinearRing
(
G
.
posList
(
' '
.
join
(
_reverse_geojson_coords
(
ll
)
for
ll
in
ring
)
)
)
for
ring
in
gj
[
'coordinates'
]
]
tag
.
append
(
G
.
exterior
(
rings
.
pop
(
0
)
)
)
for
ring
in
rings
:
tag
.
append
(
G
.
interior
(
ring
)
)
elif
gj
[
'type'
]
in
(
'MultiPoint'
,
'MultiLineString'
,
'MultiPolygon'
)
:
single_type
=
gj
[
'type'
]
[
5
:
]
member_tag
=
single_type
[
0
]
.
lower
(
)
+
single_type
[
1
:
]
+
'Member'
for
coord
in
gj
[
'coordinates'
]
:
tag
.
append
(
G
(
member_tag
,
geojson_to_gml
(
{
'type'
:
single_type
,
'coordinates'
:
coord
}
,
set_srs
=
False
)
)
)
else
:
raise
NotImplementedError
return
tag

Given a dict deserialized from a GeoJSON object returns an lxml Element of the corresponding GML geometry .


def geom_to_xml_element(geom):
    """Transform a GEOS or OGR geometry object into an lxml Element
    for the GML geometry."""
    if geom.srs.srid != 4326:
        raise NotImplementedError("Only WGS 84 lat/long geometries (SRID 4326) are supported.")
    # GeoJSON output is far more standard than GML, so go through that
    return geojson_to_gml(json.loads(geom.geojson))

Transform a GEOS or OGR geometry object into an lxml Element for the GML geometry .


def remove_comments(tex_source):
    """Delete latex comments from TeX source.

    Parameters
    ----------
    tex_source : str
        TeX source content.

    Returns
    -------
    tex_source : str
        TeX source without comments.
    """
    # Expression via http://stackoverflow.com/a/13365453
    return re.sub(r'(?<!\\)%.*$', r'', tex_source, flags=re.M)


def replace_macros(tex_source, macros):
    r"""Replace macros in the TeX source with their content.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros. See
        `lsstprojectmeta.tex.scraper.get_macros`.

    Returns
    -------
    tex_source : `str`
        TeX source with known macros replaced.

    Notes
    -----
    Macros with arguments are not supported.

    Examples
    --------
    >>> macros = {r'\handle': 'LDM-nnn'}
    >>> sample = r'This is document \handle.'
    >>> replace_macros(sample, macros)
    'This is document LDM-nnn.'

    Any trailing slash after the macro command is also replaced by this
    function.

    >>> macros = {r'\product': 'Data Management'}
    >>> sample = r'\title    [Test Plan]  { \product\ Test Plan}'
    >>> replace_macros(sample, macros)
    '\\title    [Test Plan]  { Data Management Test Plan}'
    """
    for macro_name, macro_content in macros.items():
        # '\\?' suffix matches an optional trailing '\' that might be used
        # for spacing.
        pattern = re.escape(macro_name) + r"\\?"
        # Wrap macro_content in lambda to avoid processing escapes
        tex_source = re.sub(pattern, lambda _: macro_content, tex_source)
    return tex_source

r Replace macros in the TeX source with their content .


def ensure_format(doc, format):
    """
    Ensures that the provided document is an lxml Element or json dict.
    """
    assert format in ('xml', 'json')
    if getattr(doc, 'tag', None) == 'open511':
        if format == 'json':
            return xml_to_json(doc)
    elif isinstance(doc, dict) and 'meta' in doc:
        if format == 'xml':
            return json_doc_to_xml(doc)
    else:
        raise ValueError("Unrecognized input document")
    return doc

def
ensure_format
(
doc
,
format
)
:
assert
format
in
(
'xml'
,
'json'
)
if
getattr
(
doc
,
'tag'
,
)
==
'open511'
:
if
format
==
'json'
:
return
xml_to_json
(
doc
)
elif
isinstance
(
doc
,
dict
)
and
'meta'
in
doc
:
if
format
==
'xml'
:
return
json_doc_to_xml
(
doc
)
else
:
raise
ValueError
(
"Unrecognized input document"
)
return
doc

Ensures that the provided document is an lxml Element or json dict .


def open511_convert(input_doc, output_format, serialize=True, **kwargs):
    """
    Convert an Open511 document between formats.
    input_doc - either an lxml open511 Element or a deserialized JSON dict
    output_format - short string name of a valid output format, as listed above
    """

    try:
        output_format_info = FORMATS[output_format]
    except KeyError:
        raise ValueError("Unrecognized output format %s" % output_format)

    input_doc = ensure_format(input_doc, output_format_info.input_format)

    result = output_format_info.func(input_doc, **kwargs)
    if serialize:
        result = output_format_info.serializer(result)
    return result

def
open511_convert
(
input_doc
,
output_format
,
serialize
=
True
,
*
*
kwargs
)
:
try
:
output_format_info
=
FORMATS
[
output_format
]
except
KeyError
:
raise
ValueError
(
"Unrecognized output format %s"
%
output_format
)
input_doc
=
ensure_format
(
input_doc
,
output_format_info
.
input_format
)
result
=
output_format_info
.
func
(
input_doc
,
*
*
kwargs
)
if
serialize
:
result
=
output_format_info
.
serializer
(
result
)
return
result

Convert an Open511 document between formats . input_doc - either an lxml open511 Element or a deserialized JSON dict output_format - short string name of a valid output format as listed above


def read(cls, root_tex_path):
        """Construct an `LsstLatexDoc` instance by reading and parsing the
        LaTeX source.

        Parameters
        ----------
        root_tex_path : `str`
            Path to the LaTeX source on the filesystem. For multi-file LaTeX
            projects this should be the path to the root document.

        Notes
        -----
        This method implements the following pipeline:

        1. `lsstprojectmeta.tex.normalizer.read_tex_file`
        2. `lsstprojectmeta.tex.scraper.get_macros`
        3. `lsstprojectmeta.tex.normalizer.replace_macros`

        Thus ``input`` and ``includes`` are resolved along with simple macros.
        """
        # Read and normalize the TeX source, replacing macros with content
        root_dir = os.path.dirname(root_tex_path)
        tex_source = read_tex_file(root_tex_path)
        tex_macros = get_macros(tex_source)
        tex_source = replace_macros(tex_source, tex_macros)
        return cls(tex_source, root_dir=root_dir)

Construct an LsstLatexDoc instance by reading and parsing the LaTeX source .


def format_content(self, format='plain', mathjax=False,
                       smart=True, extra_args=None):
        """Get the document content in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content.
        """
        output_text = convert_lsstdoc_tex(
            self._tex, format,
            mathjax=mathjax,
            smart=smart,
            extra_args=extra_args)
        return output_text

def format_content ( self , format = 'plain' , mathjax = False , smart = True , extra_args = None ) : output_text = convert_lsstdoc_tex ( self . _tex , format , mathjax = mathjax , smart = smart , extra_args = extra_args ) return output_text

Get the document content in the specified markup format .


def format_title(self, format='html5', deparagraph=True, mathjax=False,
                     smart=True, extra_args=None):
        """Get the document title in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the title is not available in
            the document.
        """
        if self.title is None:
            return None

        output_text = convert_lsstdoc_tex(
            self.title, format,
            deparagraph=deparagraph,
            mathjax=mathjax,
            smart=smart,
            extra_args=extra_args)
        return output_text

def format_title ( self , format = 'html5' , deparagraph = True , mathjax = False , smart = True , extra_args = None ) : if self . title is None : return None output_text = convert_lsstdoc_tex ( self . title , format , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args ) return output_text

Get the document title in the specified markup format .


def format_short_title(self, format='html5', deparagraph=True,
                           mathjax=False, smart=True, extra_args=None):
        """Get the document short title in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the short title is not available in
            the document.
        """
        if self.short_title is None:
            return None

        output_text = convert_lsstdoc_tex(
            self.short_title, 'html5',
            deparagraph=deparagraph,
            mathjax=mathjax,
            smart=smart,
            extra_args=extra_args)
        return output_text

def format_short_title ( self , format = 'html5' , deparagraph = True , mathjax = False , smart = True , extra_args = None ) : if self . short_title is None : return None output_text = convert_lsstdoc_tex ( self . short_title , 'html5' , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args ) return output_text

Get the document short title in the specified markup format .


def format_abstract(self, format='html5', deparagraph=False, mathjax=False,
                        smart=True, extra_args=None):
        """Get the document abstract in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the title is not available in
            the document.
        """
        if self.abstract is None:
            return None

        abstract_latex = self._prep_snippet_for_pandoc(self.abstract)

        output_text = convert_lsstdoc_tex(
            abstract_latex, format,
            deparagraph=deparagraph,
            mathjax=mathjax,
            smart=smart,
            extra_args=extra_args)
        return output_text

def format_abstract ( self , format = 'html5' , deparagraph = False , mathjax = False , smart = True , extra_args = None ) : if self . abstract is None : return None abstract_latex = self . _prep_snippet_for_pandoc ( self . abstract ) output_text = convert_lsstdoc_tex ( abstract_latex , format , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args ) return output_text

Get the document abstract in the specified markup format .


def format_authors(self, format='html5', deparagraph=True, mathjax=False,
                       smart=True, extra_args=None):
        """Get the document authors in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `list` of `str`
            Sequence of author names in the specified output markup format.
        """
        formatted_authors = []
        for latex_author in self.authors:
            formatted_author = convert_lsstdoc_tex(
                latex_author, format,
                deparagraph=deparagraph,
                mathjax=mathjax,
                smart=smart,
                extra_args=extra_args)
            # removes Pandoc's terminal newlines
            formatted_author = formatted_author.strip()
            formatted_authors.append(formatted_author)
        return formatted_authors

def
format_authors
(
self
,
format
=
'html5'
,
deparagraph
=
True
,
mathjax
=
False
,
smart
=
True
,
extra_args
=
)
:
formatted_authors
=
[
]
for
latex_author
in
self
.
authors
:
formatted_author
=
convert_lsstdoc_tex
(
latex_author
,
format
,
deparagraph
=
deparagraph
,
mathjax
=
mathjax
,
smart
=
smart
,
extra_args
=
extra_args
)
# removes Pandoc's terminal newlines
formatted_author
=
formatted_author
.
strip
(
)
formatted_authors
.
append
(
formatted_author
)
return
formatted_authors

Get the document authors in the specified markup format .


def _parse_documentclass(self):
        """Parse documentclass options.

        Sets the the ``_document_options`` attribute.
        """
        command = LatexCommand(
            'documentclass',
            {'name': 'options', 'required': False, 'bracket': '['},
            {'name': 'class_name', 'required': True, 'bracket': '{'})
        try:
            parsed = next(command.parse(self._tex))
        except StopIteration:
            self._logger.warning('lsstdoc has no documentclass')
            self._document_options = []

        try:
            content = parsed['options']
            self._document_options = [opt.strip()
                                      for opt in content.split(',')]
        except KeyError:
            self._logger.warning('lsstdoc has no documentclass options')
            self._document_options = []

def
_parse_documentclass
(
self
)
:
command
=
LatexCommand
(
'documentclass'
,
{
'name'
:
'options'
,
'required'
:
False
,
'bracket'
:
'['
}
,
{
'name'
:
'class_name'
,
'required'
:
True
,
'bracket'
:
'{'
}
)
try
:
parsed
=
next
(
command
.
parse
(
self
.
_tex
)
)
except
StopIteration
:
self
.
_logger
.
warning
(
'lsstdoc has no documentclass'
)
self
.
_document_options
=
[
]
try
:
content
=
parsed
[
'options'
]
self
.
_document_options
=
[
opt
.
strip
(
)
for
opt
in
content
.
split
(
','
)
]
except
KeyError
:
self
.
_logger
.
warning
(
'lsstdoc has no documentclass options'
)
self
.
_document_options
=
[
]


def _parse_title(self):
        """Parse the title from TeX source.

        Sets these attributes:

        - ``_title``
        - ``_short_title``
        """
        command = LatexCommand(
            'title',
            {'name': 'short_title', 'required': False, 'bracket': '['},
            {'name': 'long_title', 'required': True, 'bracket': '{'})
        try:
            parsed = next(command.parse(self._tex))
        except StopIteration:
            self._logger.warning('lsstdoc has no title')
            self._title = None
            self._short_title = None

        self._title = parsed['long_title']

        try:
            self._short_title = parsed['short_title']
        except KeyError:
            self._logger.warning('lsstdoc has no short title')
            self._short_title = None

def
_parse_title
(
self
)
:
command
=
LatexCommand
(
'title'
,
{
'name'
:
'short_title'
,
'required'
:
False
,
'bracket'
:
'['
}
,
{
'name'
:
'long_title'
,
'required'
:
True
,
'bracket'
:
'{'
}
)
try
:
parsed
=
next
(
command
.
parse
(
self
.
_tex
)
)
except
StopIteration
:
self
.
_logger
.
warning
(
'lsstdoc has no title'
)
self
.
_title
=
self
.
_short_title
=
self
.
_title
=
parsed
[
'long_title'
]
try
:
self
.
_short_title
=
parsed
[
'short_title'
]
except
KeyError
:
self
.
_logger
.
warning
(
'lsstdoc has no short title'
)
self
.
_short_title
=


def _parse_doc_ref(self):
        """Parse the document handle.

        Sets the ``_series``, ``_serial``, and ``_handle`` attributes.
        """
        command = LatexCommand(
            'setDocRef',
            {'name': 'handle', 'required': True, 'bracket': '{'})
        try:
            parsed = next(command.parse(self._tex))
        except StopIteration:
            self._logger.warning('lsstdoc has no setDocRef')
            self._handle = None
            self._series = None
            self._serial = None
            return

        self._handle = parsed['handle']
        try:
            self._series, self._serial = self._handle.split('-', 1)
        except ValueError:
            self._logger.warning('lsstdoc handle cannot be parsed into '
                                 'series and serial: %r', self._handle)
            self._series = None
            self._serial = None

def
_parse_doc_ref
(
self
)
:
command
=
LatexCommand
(
'setDocRef'
,
{
'name'
:
'handle'
,
'required'
:
True
,
'bracket'
:
'{'
}
)
try
:
parsed
=
next
(
command
.
parse
(
self
.
_tex
)
)
except
StopIteration
:
self
.
_logger
.
warning
(
'lsstdoc has no setDocRef'
)
self
.
_handle
=
self
.
_series
=
self
.
_serial
=
return
self
.
_handle
=
parsed
[
'handle'
]
try
:
self
.
_series
,
self
.
_serial
=
self
.
_handle
.
split
(
'-'
,
1
)
except
ValueError
:
self
.
_logger
.
warning
(
'lsstdoc handle cannot be parsed into '
'series and serial: %r'
,
self
.
_handle
)
self
.
_series
=
self
.
_serial
=


def _parse_author(self):
        r"""Parse the author from TeX source.

        Sets the ``_authors`` attribute.

        Goal is to parse::

           \author{
           A.~Author,
           B.~Author,
           and
           C.~Author}

        Into::

           ['A. Author', 'B. Author', 'C. Author']
        """
        command = LatexCommand(
            'author',
            {'name': 'authors', 'required': True, 'bracket': '{'})
        try:
            parsed = next(command.parse(self._tex))
        except StopIteration:
            self._logger.warning('lsstdoc has no author')
            self._authors = []
            return

        try:
            content = parsed['authors']
        except KeyError:
            self._logger.warning('lsstdoc has no author')
            self._authors = []
            return

        # Clean content
        content = content.replace('\n', ' ')
        content = content.replace('~', ' ')
        content = content.strip()

        # Split content into list of individual authors
        authors = []
        for part in content.split(','):
            part = part.strip()
            for split_part in part.split('and '):
                split_part = split_part.strip()
                if len(split_part) > 0:
                    authors.append(split_part)
        self._authors = authors


def _parse_abstract(self):
        """Parse the abstract from the TeX source.

        Sets the ``_abstract`` attribute.
        """
        command = LatexCommand(
            'setDocAbstract',
            {'name': 'abstract', 'required': True, 'bracket': '{'})
        try:
            parsed = next(command.parse(self._tex))
        except StopIteration:
            self._logger.warning('lsstdoc has no abstract')
            self._abstract = None
            return

        try:
            content = parsed['abstract']
        except KeyError:
            self._logger.warning('lsstdoc has no abstract')
            self._abstract = None
            return

        content = content.strip()
        self._abstract = content

def
_parse_abstract
(
self
)
:
command
=
LatexCommand
(
'setDocAbstract'
,
{
'name'
:
'abstract'
,
'required'
:
True
,
'bracket'
:
'{'
}
)
try
:
parsed
=
next
(
command
.
parse
(
self
.
_tex
)
)
except
StopIteration
:
self
.
_logger
.
warning
(
'lsstdoc has no abstract'
)
self
.
_abstract
=
return
try
:
content
=
parsed
[
'abstract'
]
except
KeyError
:
self
.
_logger
.
warning
(
'lsstdoc has no abstract'
)
self
.
_abstract
=
return
content
=
content
.
strip
(
)
self
.
_abstract
=
content

Parse the abstract from the TeX source .


def _prep_snippet_for_pandoc(self, latex_text):
        """Process a LaTeX snippet of content for better transformation
        with pandoc.

        Currently runs the CitationLinker to convert BibTeX citations to
        href links.
        """
        replace_cite = CitationLinker(self.bib_db)
        latex_text = replace_cite(latex_text)
        return latex_text

def _prep_snippet_for_pandoc ( self , latex_text ) : replace_cite = CitationLinker ( self . bib_db ) latex_text = replace_cite ( latex_text ) return latex_text

Process a LaTeX snippet of content for better transformation with pandoc .


def _load_bib_db(self):
        r"""Load the BibTeX bibliography referenced by the document.

        This method triggered by the `bib_db` attribute and populates the
        `_bib_db` private attribute.

        The ``\bibliography`` command is parsed to identify the bibliographies
        referenced by the document.
        """
        # Get the names of custom bibtex files by parsing the
        # \bibliography command and filtering out the default lsstdoc
        # bibliographies.
        command = LatexCommand(
            'bibliography',
            {'name': 'bib_names', 'required': True, 'bracket': '{'})
        try:
            parsed = next(command.parse(self._tex))
            bib_names = [n.strip() for n in parsed['bib_names'].split(',')]
        except StopIteration:
            self._logger.warning('lsstdoc has no bibliography command')
            bib_names = []
        custom_bib_names = [n for n in bib_names
                            if n not in KNOWN_LSSTTEXMF_BIB_NAMES]

        # Read custom bibliographies.
        custom_bibs = []
        for custom_bib_name in custom_bib_names:
            custom_bib_path = os.path.join(
                os.path.join(self._root_dir),
                custom_bib_name + '.bib'
            )
            if not os.path.exists(custom_bib_path):
                self._logger.warning('Could not find bibliography %r',
                                     custom_bib_path)
                continue
            with open(custom_bib_path, 'r') as file_handle:
                custom_bibs.append(file_handle.read())
        if len(custom_bibs) > 0:
            custom_bibtex = '\n\n'.join(custom_bibs)
        else:
            custom_bibtex = None

        # Get the combined pybtex bibliography
        db = get_bibliography(bibtex=custom_bibtex)

        self._bib_db = db

r Load the BibTeX bibliography referenced by the document .


def _parse_revision_date(self):
        r"""Parse the ``\date`` command, falling back to getting the
        most recent Git commit date and the current datetime.

        Result is available from the `revision_datetime` attribute.
        """
        doc_datetime = None

        # First try to parse the \date command in the latex.
        # \date is ignored for draft documents.
        if not self.is_draft:
            date_command = LatexCommand(
                'date',
                {'name': 'content', 'required': True, 'bracket': '{'})
            try:
                parsed = next(date_command.parse(self._tex))
                command_content = parsed['content'].strip()
            except StopIteration:
                command_content = None
                self._logger.warning('lsstdoc has no date command')

            # Try to parse a date from the \date command
            if command_content is not None and command_content != r'\today':
                try:
                    doc_datetime = datetime.datetime.strptime(command_content,
                                                              '%Y-%m-%d')
                    # Assume LSST project time (Pacific)
                    project_tz = timezone('US/Pacific')
                    localized_datetime = project_tz.localize(doc_datetime)
                    # Normalize to UTC
                    doc_datetime = localized_datetime.astimezone(pytz.utc)

                    self._revision_datetime_source = 'tex'
                except ValueError:
                    self._logger.warning('Could not parse a datetime from '
                                         'lsstdoc date command: %r',
                                         command_content)

        # Fallback to getting the datetime from Git
        if doc_datetime is None:
            content_extensions = ('tex', 'bib', 'pdf', 'png', 'jpg')
            try:
                doc_datetime = get_content_commit_date(
                    content_extensions,
                    root_dir=self._root_dir)
                self._revision_datetime_source = 'git'
            except RuntimeError:
                self._logger.warning('Could not get a datetime from the Git '
                                     'repository at %r',
                                     self._root_dir)

        # Final fallback to the current datetime
        if doc_datetime is None:
            doc_datetime = pytz.utc.localize(datetime.datetime.now())
            self._revision_datetime_source = 'now'

        self._datetime = doc_datetime

r Parse the \ date command falling back to getting the most recent Git commit date and the current datetime .


def build_jsonld(self, url=None, code_url=None, ci_url=None,
                     readme_url=None, license_id=None):
        """Create a JSON-LD representation of this LSST LaTeX document.

        Parameters
        ----------
        url : `str`, optional
            URL where this document is published to the web. Prefer
            the LSST the Docs URL if possible.
            Example: ``'https://ldm-151.lsst.io'``.
        code_url : `str`, optional
            Path the the document's repository, typically on GitHub.
            Example: ``'https://github.com/lsst/LDM-151'``.
        ci_url : `str`, optional
            Path to the continuous integration service dashboard for this
            document's repository.
            Example: ``'https://travis-ci.org/lsst/LDM-151'``.
        readme_url : `str`, optional
            URL to the document repository's README file. Example:
            ``https://raw.githubusercontent.com/lsst/LDM-151/master/README.rst``.
        license_id : `str`, optional
            License identifier, if known. The identifier should be from the
            listing at https://spdx.org/licenses/. Example: ``CC-BY-4.0``.

        Returns
        -------
        jsonld : `dict`
            JSON-LD-formatted dictionary.
        """
        jsonld = {
            '@context': [
                "https://raw.githubusercontent.com/codemeta/codemeta/2.0-rc/"
                "codemeta.jsonld",
                "http://schema.org"],
            '@type': ['Report', 'SoftwareSourceCode'],
            'language': 'TeX',
            'reportNumber': self.handle,
            'name': self.plain_title,
            'description': self.plain_abstract,
            'author': [{'@type': 'Person', 'name': author_name}
                       for author_name in self.plain_authors],
            # This is a datetime.datetime; not a string. If writing to a file,
            # Need to convert this to a ISO 8601 string.
            'dateModified': self.revision_datetime
        }

        try:
            jsonld['articleBody'] = self.plain_content
            jsonld['fileFormat'] = 'text/plain'  # MIME type of articleBody
        except RuntimeError:
            # raised by pypandoc when it can't convert the tex document
            self._logger.exception('Could not convert latex body to plain '
                                   'text for articleBody.')
            self._logger.warning('Falling back to tex source for articleBody')
            jsonld['articleBody'] = self._tex
            jsonld['fileFormat'] = 'text/plain'  # no mimetype for LaTeX?

        if url is not None:
            jsonld['@id'] = url
            jsonld['url'] = url
        else:
            # Fallback to using the document handle as the ID. This isn't
            # entirely ideal from a linked data perspective.
            jsonld['@id'] = self.handle

        if code_url is not None:
            jsonld['codeRepository'] = code_url

        if ci_url is not None:
            jsonld['contIntegration'] = ci_url

        if readme_url is not None:
            jsonld['readme'] = readme_url

        if license_id is not None:
            jsonld['license_id'] = None

        return jsonld

Create a JSON - LD representation of this LSST LaTeX document .


def rename(self, from_name, to_name):
        """Renames an existing database."""
        log.info('renaming database from %s to %s' % (from_name, to_name))
        self._run_stmt('alter database %s rename to %s' % (from_name, to_name))

def
rename
(
self
,
from_name
,
to_name
)
:
log
.
info
(
'renaming database from %s to %s'
%
(
from_name
,
to_name
)
)
self
.
_run_stmt
(
'alter database %s rename to %s'
%
(
from_name
,
to_name
)
)


def available(self, timeout=5):
        """Returns True if database server is running, False otherwise."""
        host = self._connect_args['host']
        port = self._connect_args['port']
        try:
            sock = socket.create_connection((host, port), timeout=timeout)
            sock.close()
            return True
        except socket.error:
            pass
        return False

def available ( self , timeout = 5 ) : host = self . _connect_args [ 'host' ] port = self . _connect_args [ 'port' ] try : sock = socket . create_connection ( ( host , port ) , timeout = timeout ) sock . close ( ) return True except socket . error : pass return False

Returns True if database server is running False otherwise .


def dump(self, name, filename):
        """
        Saves the state of a database to a file.

        Parameters
        ----------
        name: str
            the database to be backed up.
        filename: str
            path to a file where database backup will be written.
        """
        if not self.exists(name):
            raise DatabaseError('database %s does not exist!')
        log.info('dumping %s to %s' % (name, filename))
        self._run_cmd('pg_dump', '--verbose', '--blobs', '--format=custom',
                      '--file=%s' % filename, name)

def
dump
(
self
,
name
,
filename
)
:
if
not
self
.
exists
(
name
)
:
raise
DatabaseError
(
'database %s does not exist!'
)
log
.
info
(
'dumping %s to %s'
%
(
name
,
filename
)
)
self
.
_run_cmd
(
'pg_dump'
,
'--verbose'
,
'--blobs'
,
'--format=custom'
,
'--file=%s'
%
filename
,
name
)

Saves the state of a database to a file .


def restore(self, name, filename):
        """
        Loads state of a backup file to a database.

        Note
        ----
        If database name does not exist, it will be created.

        Parameters
        ----------
        name: str
            the database to which backup will be restored.
        filename: str
            path to a file contain a postgres database backup.
        """
        if not self.exists(name):
            self.create(name)
        else:
            log.warn('overwriting contents of database %s' % name)
        log.info('restoring %s from %s' % (name, filename))
        self._run_cmd('pg_restore', '--verbose', '--dbname=%s' % name, filename)

def
restore
(
self
,
name
,
filename
)
:
if
not
self
.
exists
(
name
)
:
self
.
create
(
name
)
else
:
log
.
warn
(
'overwriting contents of database %s'
%
name
)
log
.
info
(
'restoring %s from %s'
%
(
name
,
filename
)
)
self
.
_run_cmd
(
'pg_restore'
,
'--verbose'
,
'--dbname=%s'
%
name
,
filename
)

Loads state of a backup file to a database .


def connection_dsn(self, name=None):
        """
        Provides a connection string for database.

        Parameters
        ----------
        name: str, optional
            an override database name for the connection string.

        Returns
        -------
        str: the connection string (e.g. 'dbname=db1 user=user1 host=localhost port=5432')
        """
        return ' '.join("%s=%s" % (param, value) for param, value in self._connect_options(name))

def
connection_dsn
(
self
,
name
=
)
:
return
' '
.
join
(
"%s=%s"
%
(
param
,
value
)
for
param
,
value
in
self
.
_connect_options
(
name
)
)

Provides a connection string for database .


def connection_url(self, name=None):
        """
        Provides a connection string for database as a sqlalchemy compatible URL.

        NB - this doesn't include special arguments related to SSL connectivity (which are outside the scope
        of the connection URL format).

        Parameters
        ----------
        name: str, optional
            an override database name for the connection string.

        Returns
        -------
        str: the connection URL (e.g. postgresql://user1@localhost:5432/db1)
            """
        return 'postgresql://{user}@{host}:{port}/{dbname}'.format(**{k: v for k, v in self._connect_options(name)})

Provides a connection string for database as a sqlalchemy compatible URL .


def shell(self, expect=pexpect):
        """
        Connects the database client shell to the database.

        Parameters
        ----------
        expect_module: str
            the database to which backup will be restored.
        """
        dsn = self.connection_dsn()
        log.debug('connection string: %s' % dsn)
        child = expect.spawn('psql "%s"' % dsn)
        if self._connect_args['password'] is not None:
            child.expect('Password: ')
            child.sendline(self._connect_args['password'])
        child.interact()

def
shell
(
self
,
expect
=
pexpect
)
:
dsn
=
self
.
connection_dsn
(
)
log
.
debug
(
'connection string: %s'
%
dsn
)
child
=
expect
.
spawn
(
'psql "%s"'
%
dsn
)
if
self
.
_connect_args
[
'password'
]
is
not
:
child
.
expect
(
'Password: '
)
child
.
sendline
(
self
.
_connect_args
[
'password'
]
)
child
.
interact
(
)

Connects the database client shell to the database .


def settings(self):
        """Returns settings from the server."""
        stmt = "select {fields} from pg_settings".format(fields=', '.join(SETTINGS_FIELDS))
        settings = []
        for row in self._iter_results(stmt):
            row['setting'] = self._vartype_map[row['vartype']](row['setting'])
            settings.append(Settings(**row))
        return settings

def
settings
(
self
)
:
stmt
=
"select {fields} from pg_settings"
.
format
(
fields
=
', '
.
join
(
SETTINGS_FIELDS
)
)
settings
=
[
]
for
row
in
self
.
_iter_results
(
stmt
)
:
row
[
'setting'
]
=
self
.
_vartype_map
[
row
[
'vartype'
]
]
(
row
[
'setting'
]
)
settings
.
append
(
Settings
(
*
*
row
)
)
return
settings


def breakfast(self, message="Breakfast is ready", shout: bool = False):
        """Say something in the morning"""
        return self.helper.output(message, shout)

def
breakfast
(
self
,
message
=
"Breakfast is ready"
,
shout
:
bool
=
False
)
:
return
self
.
helper
.
output
(
message
,
shout
)


def lunch(self, message="Time for lunch", shout: bool = False):
        """Say something in the afternoon"""
        return self.helper.output(message, shout)

def
lunch
(
self
,
message
=
"Time for lunch"
,
shout
:
bool
=
False
)
:
return
self
.
helper
.
output
(
message
,
shout
)


def dinner(self, message="Dinner is served", shout: bool = False):
        """Say something in the evening"""
        return self.helper.output(message, shout)

def
dinner
(
self
,
message
=
"Dinner is served"
,
shout
:
bool
=
False
)
:
return
self
.
helper
.
output
(
message
,
shout
)