def main():
    """Command line entrypoint to reduce technote metadata.
    """
    parser = argparse.ArgumentParser(
        description='Discover and ingest metadata from document sources, '
                    'including lsstdoc-based LaTeX documents and '
                    'reStructuredText-based technotes. Metadata can be '
                    'upserted into the LSST Projectmeta MongoDB.')
    parser.add_argument(
        '--ltd-product',
        dest='ltd_product_url',
        help='URL of an LSST the Docs product '
             '(https://keeper.lsst.codes/products/<slug>). If provided, '
             'only this document will be ingested.')
    parser.add_argument(
        '--github-token',
        help='GitHub personal access token.')
    parser.add_argument(
        '--mongodb-uri',
        help='MongoDB connection URI. If provided, metadata will be loaded '
             'into the Projectmeta database. Omit this argument to just '
             'test the ingest pipeline.')
    parser.add_argument(
        '--mongodb-db',
        default='lsstprojectmeta',
        help='Name of MongoDB database')
    parser.add_argument(
        '--mongodb-collection',
        default='resources',
        help='Name of the MongoDB collection for projectmeta resources')
    args = parser.parse_args()

    # Configure the root logger
    stream_handler = logging.StreamHandler()
    stream_formatter = logging.Formatter(
        '%(asctime)s %(levelname)8s %(name)s | %(message)s')
    stream_handler.setFormatter(stream_formatter)
    root_logger = logging.getLogger()
    root_logger.addHandler(stream_handler)
    root_logger.setLevel(logging.WARNING)
    # Configure app logger
    app_logger = logging.getLogger('lsstprojectmeta')
    app_logger.setLevel(logging.DEBUG)

    if args.mongodb_uri is not None:
        mongo_client = AsyncIOMotorClient(args.mongodb_uri, ssl=True)
        collection = mongo_client[args.mongodb_db][args.mongodb_collection]
    else:
        collection = None

    loop = asyncio.get_event_loop()

    if args.ltd_product_url is not None:
        # Run single technote
        loop.run_until_complete(run_single_ltd_doc(args.ltd_product_url,
                                                   args.github_token,
                                                   collection))
    else:
        # Run bulk technote processing
        loop.run_until_complete(run_bulk_etl(args.github_token,
                                             collection))

Command line entrypoint to reduce technote metadata .


async def process_ltd_doc_products(session, product_urls, github_api_token,
                                   mongo_collection=None):
    """Run a pipeline to process extract, transform, and load metadata for
    multiple LSST the Docs-hosted projects

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    product_urls : `list` of `str`
        List of LSST the Docs product URLs.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records.
    """
    tasks = [asyncio.ensure_future(
             process_ltd_doc(session, github_api_token,
                             product_url,
                             mongo_collection=mongo_collection))
             for product_url in product_urls]
    await asyncio.gather(*tasks)

async def process_ltd_doc_products ( session , product_urls , github_api_token , mongo_collection = None ) : tasks = [ asyncio . ensure_future ( process_ltd_doc ( session , github_api_token , product_url , mongo_collection = mongo_collection ) ) for product_url in product_urls ] await asyncio . gather ( * tasks )

Run a pipeline to process extract transform and load metadata for multiple LSST the Docs - hosted projects


async def process_ltd_doc(session, github_api_token, ltd_product_url,
                          mongo_collection=None):
    """Ingest any kind of LSST document hosted on LSST the Docs from its
    source.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    ltd_product_url : `str`
        URL of the technote's product resource in the LTD Keeper API.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records. If provided, ths JSON-LD is upserted
        into the MongoDB collection.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    .. `GitHub personal access token guide`: https://ls.st/41d
    """
    logger = logging.getLogger(__name__)

    ltd_product_data = await get_ltd_product(session, url=ltd_product_url)

    # Ensure the LTD product is a document
    product_name = ltd_product_data['slug']
    doc_handle_match = DOCUMENT_HANDLE_PATTERN.match(product_name)
    if doc_handle_match is None:
        logger.debug('%s is not a document repo', product_name)
        return

    # Figure out the format of the document by probing for metadata files.
    # reStructuredText-based Sphinx documents have metadata.yaml file.
    try:
        return await process_sphinx_technote(session,
                                             github_api_token,
                                             ltd_product_data,
                                             mongo_collection=mongo_collection)
    except NotSphinxTechnoteError:
        # Catch error so we can try the next format
        logger.debug('%s is not a Sphinx-based technote.', product_name)
    except Exception:
        # Something bad happened trying to process the technote.
        # Log and just move on.
        logger.exception('Unexpected error trying to process %s', product_name)
        return

    # Try interpreting it as a Lander page with a /metadata.jsonld document
    try:
        return await process_lander_page(session,
                                         github_api_token,
                                         ltd_product_data,
                                         mongo_collection=mongo_collection)
    except NotLanderPageError:
        # Catch error so we can try the next format
        logger.debug('%s is not a Lander page with a metadata.jsonld file.',
                     product_name)
    except Exception:
        # Something bad happened; log and move on
        logger.exception('Unexpected error trying to process %s', product_name)
        return

Ingest any kind of LSST document hosted on LSST the Docs from its source .


def decorator(decorator_func):
    """Allows a decorator to be called with or without keyword arguments."""
    assert callable(decorator_func), type(decorator_func)

    def _decorator(func=None, **kwargs):
        assert func is None or callable(func), type(func)
        if func:
            return decorator_func(func, **kwargs)
        else:
            def _decorator_helper(func):
                return decorator_func(func, **kwargs)

            return _decorator_helper

    return _decorator

def decorator ( decorator_func ) : assert callable ( decorator_func ) , type ( decorator_func ) def _decorator ( func = None , * * kwargs ) : assert func is None or callable ( func ) , type ( func ) if func : return decorator_func ( func , * * kwargs ) else : def _decorator_helper ( func ) : return decorator_func ( func , * * kwargs ) return _decorator_helper return _decorator

Allows a decorator to be called with or without keyword arguments .


def get_installation_token(installation_id, integration_jwt):
    """Create a GitHub token for an integration installation.

    Parameters
    ----------
    installation_id : `int`
        Installation ID. This is available in the URL of the integration's
        **installation** ID.
    integration_jwt : `bytes`
        The integration's JSON Web Token (JWT). You can create this with
        `create_jwt`.

    Returns
    -------
    token_obj : `dict`
        GitHub token object. Includes the fields:

        - ``token``: the token string itself.
        - ``expires_at``: date time string when the token expires.

    Example
    -------
    The typical workflow for authenticating to an integration installation is:

    .. code-block:: python

       from dochubadapter.github import auth
       jwt = auth.create_jwt(integration_id, private_key_path)
       token_obj = auth.get_installation_token(installation_id, jwt)
       print(token_obj['token'])

    Notes
    -----
    See
    https://developer.github.com/early-access/integrations/authentication/#as-an-installation
    for more information
    """
    api_root = 'https://api.github.com'
    url = '{root}/installations/{id_:d}/access_tokens'.format(
        api_root=api_root,
        id_=installation_id)

    headers = {
        'Authorization': 'Bearer {0}'.format(integration_jwt.decode('utf-8')),
        'Accept': 'application/vnd.github.machine-man-preview+json'
    }

    resp = requests.post(url, headers=headers)
    resp.raise_for_status()
    return resp.json()

Create a GitHub token for an integration installation .


def create_jwt(integration_id, private_key_path):
    """Create a JSON Web Token to authenticate a GitHub Integration or
    installation.

    Parameters
    ----------
    integration_id : `int`
        Integration ID. This is available from the GitHub integration's
        homepage.
    private_key_path : `str`
        Path to the integration's private key (a ``.pem`` file).

    Returns
    -------
    jwt : `bytes`
        JSON Web Token that is good for 9 minutes.

    Notes
    -----
    The JWT is encoded with the RS256 algorithm. It includes a payload with
    fields:

    - ``'iat'``: The current time, as an `int` timestamp.
    - ``'exp'``: Expiration time, as an `int timestamp. The expiration
      time is set of 9 minutes in the future (maximum allowance is 10 minutes).
    - ``'iss'``: The integration ID (`int`).

    For more information, see
    https://developer.github.com/early-access/integrations/authentication/.
    """
    integration_id = int(integration_id)

    with open(private_key_path, 'rb') as f:
        cert_bytes = f.read()

    now = datetime.datetime.now()
    expiration_time = now + datetime.timedelta(minutes=9)
    payload = {
        # Issued at time
        'iat': int(now.timestamp()),
        # JWT expiration time (10 minute maximum)
        'exp': int(expiration_time.timestamp()),
        # Integration's GitHub identifier
        'iss': integration_id
    }

    return jwt.encode(payload, cert_bytes, algorithm='RS256')

Create a JSON Web Token to authenticate a GitHub Integration or installation .


def get_macros(tex_source):
    r"""Get all macro definitions from TeX source, supporting multiple
    declaration patterns.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.

    Returns
    -------
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros.

    Notes
    -----
    This function uses the following function to scrape macros of different
    types:

    - `get_def_macros`
    - `get_newcommand_macros`

    This macro scraping has the following caveats:

    - Macro definition (including content) must all occur on one line.
    - Macros with arguments are not supported.
    """
    macros = {}
    macros.update(get_def_macros(tex_source))
    macros.update(get_newcommand_macros(tex_source))
    return macros

def get_macros ( tex_source ) : macros = { } macros . update ( get_def_macros ( tex_source ) ) macros . update ( get_newcommand_macros ( tex_source ) ) return macros

r Get all macro definitions from TeX source supporting multiple declaration patterns .


def get_def_macros(tex_source):
    r"""Get all ``\def`` macro definition from TeX source.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.

    Returns
    -------
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros.

    Notes
    -----
    ``\def`` macros with arguments are not supported.
    """
    macros = {}
    for match in DEF_PATTERN.finditer(tex_source):
        macros[match.group('name')] = match.group('content')
    return macros

def get_def_macros ( tex_source ) : macros = { } for match in DEF_PATTERN . finditer ( tex_source ) : macros [ match . group ( 'name' ) ] = match . group ( 'content' ) return macros

r Get all \ def macro definition from TeX source .


def get_newcommand_macros(tex_source):
    r"""Get all ``\newcommand`` macro definition from TeX source.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.

    Returns
    -------
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros.

    Notes
    -----
    ``\newcommand`` macros with arguments are not supported.
    """
    macros = {}
    command = LatexCommand(
        'newcommand',
        {'name': 'name', 'required': True, 'bracket': '{'},
        {'name': 'content', 'required': True, 'bracket': '{'})

    for macro in command.parse(tex_source):
        macros[macro['name']] = macro['content']

    return macros

def get_newcommand_macros ( tex_source ) : macros = { } command = LatexCommand ( 'newcommand' , { 'name' : 'name' , 'required' : True , 'bracket' : '{' } , { 'name' : 'content' , 'required' : True , 'bracket' : '{' } ) for macro in command . parse ( tex_source ) : macros [ macro [ 'name' ] ] = macro [ 'content' ] return macros

r Get all \ newcommand macro definition from TeX source .


def load(directory_name, module_name):
    """Try to load and return a module

    Will add DIRECTORY_NAME to sys.path and tries to import MODULE_NAME.

    For example:
    load("~/.yaz", "yaz_extension")
    """
    directory_name = os.path.expanduser(directory_name)
    if os.path.isdir(directory_name) and directory_name not in sys.path:
        sys.path.append(directory_name)

    try:
        return importlib.import_module(module_name)
    except ImportError:
        pass

def load ( directory_name , module_name ) : directory_name = os . path . expanduser ( directory_name ) if os . path . isdir ( directory_name ) and directory_name not in sys . path : sys . path . append ( directory_name ) try : return importlib . import_module ( module_name ) except ImportError : pass


def make_aware(value, timezone):
    """
    Makes a naive datetime.datetime in a given time zone aware.
    """
    if hasattr(timezone, 'localize') and value not in (datetime.datetime.min, datetime.datetime.max):
        # available for pytz time zones
        return timezone.localize(value, is_dst=None)
    else:
        # may be wrong around DST changes
        return value.replace(tzinfo=timezone)

def
make_aware
(
value
,
timezone
)
:
if
hasattr
(
timezone
,
'localize'
)
and
value
not
in
(
datetime
.
datetime
.
min
,
datetime
.
datetime
.
max
)
:
# available for pytz time zones
return
timezone
.
localize
(
value
,
is_dst
=
)
else
:
# may be wrong around DST changes
return
value
.
replace
(
tzinfo
=
timezone
)

Makes a naive datetime . datetime in a given time zone aware .


def make_naive(value, timezone):
    """
    Makes an aware datetime.datetime naive in a given time zone.
    """
    value = value.astimezone(timezone)
    if hasattr(timezone, 'normalize'):
        # available for pytz time zones
        value = timezone.normalize(value)
    return value.replace(tzinfo=None)

def
make_naive
(
value
,
timezone
)
:
value
=
value
.
astimezone
(
timezone
)
if
hasattr
(
timezone
,
'normalize'
)
:
# available for pytz time zones
value
=
timezone
.
normalize
(
value
)
return
value
.
replace
(
tzinfo
=
)

Makes an aware datetime . datetime naive in a given time zone .


def to_timezone(self, dt):
        """Converts a datetime to the timezone of this Schedule."""
        if timezone.is_aware(dt):
            return dt.astimezone(self.timezone)
        else:
            return timezone.make_aware(dt, self.timezone)

def to_timezone ( self , dt ) : if timezone . is_aware ( dt ) : return dt . astimezone ( self . timezone ) else : return timezone . make_aware ( dt , self . timezone )

Converts a datetime to the timezone of this Schedule .


def next_interval(self, after=None):
        """Returns the next Period this event is in effect, or None if the event
        has no remaining periods."""
        if after is None:
            after = timezone.now()
        after = self.to_timezone(after)
        return next(self.intervals(range_start=after), None)

def next_interval ( self , after = None ) : if after is None : after = timezone . now ( ) after = self . to_timezone ( after ) return next ( self . intervals ( range_start = after ) , None )

Returns the next Period this event is in effect or None if the event has no remaining periods .


def _daily_periods(self, range_start, range_end):
        """Returns an iterator of Period tuples for every day this event is in effect, between range_start
        and range_end."""
        specific = set(self.exceptions.keys())

        return heapq.merge(self.exception_periods(range_start, range_end), *[
            sched.daily_periods(range_start=range_start, range_end=range_end, exclude_dates=specific)
            for sched in self._recurring_schedules
        ])

def _daily_periods ( self , range_start , range_end ) : specific = set ( self . exceptions . keys ( ) ) return heapq . merge ( self . exception_periods ( range_start , range_end ) , * [ sched . daily_periods ( range_start = range_start , range_end = range_end , exclude_dates = specific ) for sched in self . _recurring_schedules ] )

Returns an iterator of Period tuples for every day this event is in effect between range_start and range_end .


def intervals(self, range_start=datetime.datetime.min, range_end=datetime.datetime.max):
        """Returns an iterator of Period tuples for continuous stretches of time during
        which this event is in effect, between range_start and range_end."""

        # At the moment the algorithm works on periods split by calendar day, one at a time,
        # merging them if they're continuous; to avoid looping infinitely for infinitely long
        # periods, it splits periods as soon as they reach 60 days.
        # This algorithm could likely be improved to get rid of this restriction and improve
        # efficiency, so code should not rely on this behaviour.

        current_period = None
        max_continuous_days = 60

        range_start = self.to_timezone(range_start)
        range_end = self.to_timezone(range_end)

        for period in self._daily_periods(range_start.date(), range_end.date()):
            if period.end < range_start or period.start > range_end:
                continue
            if current_period is None:
                current_period = period
            else:
                if ( ((period.start < current_period.end)
                        or (period.start - current_period.end) <= datetime.timedelta(minutes=1))
                        and (current_period.end - current_period.start) < datetime.timedelta(days=max_continuous_days)):
                    # Merge
                    current_period = Period(current_period.start, period.end)
                else:
                    yield current_period
                    current_period = period
        if current_period:
            yield current_period

Returns an iterator of Period tuples for continuous stretches of time during which this event is in effect between range_start and range_end .


def includes(self, query_date, query_time=None):
        """Does this schedule include the provided time?
        query_date and query_time are date and time objects, interpreted
        in this schedule's timezone"""

        if self.start_date and query_date < self.start_date:
            return False
        if self.end_date and query_date > self.end_date:
            return False
        if query_date.weekday() not in self.weekdays:
            return False

        if not query_time:
            return True

        if query_time >= self.period.start and query_time <= self.period.end:
            return True

        return False

def includes ( self , query_date , query_time = None ) : if self . start_date and query_date < self . start_date : return False if self . end_date and query_date > self . end_date : return False if query_date . weekday ( ) not in self . weekdays : return False if not query_time : return True if query_time >= self . period . start and query_time <= self . period . end : return True return False

Does this schedule include the provided time? query_date and query_time are date and time objects interpreted in this schedule s timezone


def daily_periods(self, range_start=datetime.date.min, range_end=datetime.date.max, exclude_dates=tuple()):
        """Returns an iterator of Period tuples for every day this schedule is in effect, between range_start
        and range_end."""
        tz = self.timezone
        period = self.period
        weekdays = self.weekdays

        current_date = max(range_start, self.start_date)
        end_date = range_end
        if self.end_date:
            end_date = min(end_date, self.end_date)

        while current_date <= end_date:
            if current_date.weekday() in weekdays and current_date not in exclude_dates:
                yield Period(
                    tz.localize(datetime.datetime.combine(current_date, period.start)),
                    tz.localize(datetime.datetime.combine(current_date, period.end))
                )
            current_date += datetime.timedelta(days=1)

def daily_periods ( self , range_start = datetime . date . min , range_end = datetime . date . max , exclude_dates = tuple ( ) ) : tz = self . timezone period = self . period weekdays = self . weekdays current_date = max ( range_start , self . start_date ) end_date = range_end if self . end_date : end_date = min ( end_date , self . end_date ) while current_date <= end_date : if current_date . weekday ( ) in weekdays and current_date not in exclude_dates : yield Period ( tz . localize ( datetime . datetime . combine ( current_date , period . start ) ) , tz . localize ( datetime . datetime . combine ( current_date , period . end ) ) ) current_date += datetime . timedelta ( days = 1 )

Returns an iterator of Period tuples for every day this schedule is in effect between range_start and range_end .


def period(self):
        """A Period tuple representing the daily start and end time."""
        start_time = self.root.findtext('daily_start_time')
        if start_time:
            return Period(text_to_time(start_time), text_to_time(self.root.findtext('daily_end_time')))
        return Period(datetime.time(0, 0), datetime.time(23, 59))

def period ( self ) : start_time = self . root . findtext ( 'daily_start_time' ) if start_time : return Period ( text_to_time ( start_time ) , text_to_time ( self . root . findtext ( 'daily_end_time' ) ) ) return Period ( datetime . time ( 0 , 0 ) , datetime . time ( 23 , 59 ) )

A Period tuple representing the daily start and end time .


def weekdays(self):
        """A set of integers representing the weekdays the schedule recurs on,
        with Monday = 0 and Sunday = 6."""
        if not self.root.xpath('days'):
            return set(range(7))
        return set(int(d) - 1 for d in self.root.xpath('days/day/text()'))

def weekdays ( self ) : if not self . root . xpath ( 'days' ) : return set ( range ( 7 ) ) return set ( int ( d ) - 1 for d in self . root . xpath ( 'days/day/text()' ) )

A set of integers representing the weekdays the schedule recurs on with Monday = 0 and Sunday = 6 .


def temp_db(db, name=None):
    """
    A context manager that creates a temporary database.

    Useful for automated tests.

    Parameters
    ----------
    db: object
        a preconfigured DB object
    name: str, optional
        name of the database to be created. (default: globally unique name)
    """
    if name is None:
        name = temp_name()
    db.create(name)
    if not db.exists(name):
        raise DatabaseError('failed to create database %s!')
    try:
        yield name
    finally:
        db.drop(name)
        if db.exists(name):
            raise DatabaseError('failed to drop database %s!')

def
temp_db
(
db
,
name
=
)
:
if
name
is
:
name
=
temp_name
(
)
db
.
create
(
name
)
if
not
db
.
exists
(
name
)
:
raise
DatabaseError
(
'failed to create database %s!'
)
try
:
yield
name
finally
:
db
.
drop
(
name
)
if
db
.
exists
(
name
)
:
raise
DatabaseError
(
'failed to drop database %s!'
)

A context manager that creates a temporary database .


async def _download_text(url, session):
    """Asynchronously request a URL and get the encoded text content of the
    body.

    Parameters
    ----------
    url : `str`
        URL to download.
    session : `aiohttp.ClientSession`
        An open aiohttp session.

    Returns
    -------
    content : `str`
        Content downloaded from the URL.
    """
    logger = logging.getLogger(__name__)
    async with session.get(url) as response:
        # aiohttp decodes the content to a Python string
        logger.info('Downloading %r', url)
        return await response.text()

Asynchronously request a URL and get the encoded text content of the body .


async def _download_lsst_bibtex(bibtex_names):
    """Asynchronously download a set of lsst-texmf BibTeX bibliographies from
    GitHub.

    Parameters
    ----------
    bibtex_names : sequence of `str`
        Names of lsst-texmf BibTeX files to download. For example:

        .. code-block:: python

           ['lsst', 'lsst-dm', 'refs', 'books', 'refs_ads']

    Returns
    -------
    bibtexs : `list` of `str`
        List of BibTeX file content, in the same order as ``bibtex_names``.
    """
    blob_url_template = (
        'https://raw.githubusercontent.com/lsst/lsst-texmf/master/texmf/'
        'bibtex/bib/{name}.bib'
    )
    urls = [blob_url_template.format(name=name) for name in bibtex_names]

    tasks = []
    async with ClientSession() as session:
        for url in urls:
            task = asyncio.ensure_future(_download_text(url, session))
            tasks.append(task)

        return await asyncio.gather(*tasks)

Asynchronously download a set of lsst - texmf BibTeX bibliographies from GitHub .


def get_lsst_bibtex(bibtex_filenames=None):
    """Get content of lsst-texmf bibliographies.

    BibTeX content is downloaded from GitHub (``master`` branch of
    https://github.com/lsst/lsst-texmf or retrieved from an in-memory cache.

    Parameters
    ----------
    bibtex_filenames : sequence of `str`, optional
        List of lsst-texmf BibTeX files to retrieve. These can be the filenames
        of lsst-bibtex files (for example, ``['lsst.bib', 'lsst-dm.bib']``)
        or names without an extension (``['lsst', 'lsst-dm']``). The default
        (recommended) is to get *all* lsst-texmf bibliographies:

        .. code-block:: python

           ['lsst', 'lsst-dm', 'refs', 'books', 'refs_ads']

    Returns
    -------
    bibtex : `dict`
        Dictionary with keys that are bibtex file names (such as ``'lsst'``,
        ``'lsst-dm'``). Values are the corresponding bibtex file content
        (`str`).
    """
    logger = logging.getLogger(__name__)

    if bibtex_filenames is None:
        # Default lsst-texmf bibliography files
        bibtex_names = KNOWN_LSSTTEXMF_BIB_NAMES
    else:
        # Sanitize filenames (remove extensions, path)
        bibtex_names = []
        for filename in bibtex_filenames:
            name = os.path.basename(os.path.splitext(filename)[0])
            if name not in KNOWN_LSSTTEXMF_BIB_NAMES:
                logger.warning('%r is not a known lsst-texmf bib file',
                               name)
                continue
            bibtex_names.append(name)

    # names of bibtex files not in cache
    uncached_names = [name for name in bibtex_names
                      if name not in _LSSTTEXMF_BIB_CACHE]
    if len(uncached_names) > 0:
        # Download bibtex and put into the cache
        loop = asyncio.get_event_loop()
        future = asyncio.ensure_future(_download_lsst_bibtex(uncached_names))
        loop.run_until_complete(future)
        for name, text in zip(bibtex_names, future.result()):
            _LSSTTEXMF_BIB_CACHE[name] = text

    return {name: _LSSTTEXMF_BIB_CACHE[name] for name in bibtex_names}

Get content of lsst - texmf bibliographies .


def get_bibliography(lsst_bib_names=None, bibtex=None):
    """Make a pybtex BibliographyData instance from standard lsst-texmf
    bibliography files and user-supplied bibtex content.

    Parameters
    ----------
    lsst_bib_names : sequence of `str`, optional
        Names of lsst-texmf BibTeX files to include. For example:

        .. code-block:: python

           ['lsst', 'lsst-dm', 'refs', 'books', 'refs_ads']

        Default is `None`, which includes all lsst-texmf bibtex files.

    bibtex : `str`
        BibTeX source content not included in lsst-texmf. This can be content
        from a import ``local.bib`` file.

    Returns
    -------
    bibliography : `pybtex.database.BibliographyData`
        A pybtex bibliography database that includes all given sources:
        lsst-texmf bibliographies and ``bibtex``.
    """
    bibtex_data = get_lsst_bibtex(bibtex_filenames=lsst_bib_names)

    # Parse with pybtex into BibliographyData instances
    pybtex_data = [pybtex.database.parse_string(_bibtex, 'bibtex')
                   for _bibtex in bibtex_data.values()]

    # Also parse local bibtex content
    if bibtex is not None:
        pybtex_data.append(pybtex.database.parse_string(bibtex, 'bibtex'))

    # Merge BibliographyData
    bib = pybtex_data[0]
    if len(pybtex_data) > 1:
        for other_bib in pybtex_data[1:]:
            for key, entry in other_bib.entries.items():
                bib.add_entry(key, entry)

    return bib

Make a pybtex BibliographyData instance from standard lsst - texmf bibliography files and user - supplied bibtex content .


def get_url_from_entry(entry):
    """Get a usable URL from a pybtex entry.

    Parameters
    ----------
    entry : `pybtex.database.Entry`
        A pybtex bibliography entry.

    Returns
    -------
    url : `str`
        Best available URL from the ``entry``.

    Raises
    ------
    NoEntryUrlError
        Raised when no URL can be made from the bibliography entry.

    Notes
    -----
    The order of priority is:

    1. ``url`` field
    2. ``ls.st`` URL from the handle for ``@docushare`` entries.
    3. ``adsurl``
    4. DOI
    """
    if 'url' in entry.fields:
        return entry.fields['url']
    elif entry.type.lower() == 'docushare':
        return 'https://ls.st/' + entry.fields['handle']
    elif 'adsurl' in entry.fields:
        return entry.fields['adsurl']
    elif 'doi' in entry.fields:
        return 'https://doi.org/' + entry.fields['doi']
    else:
        raise NoEntryUrlError()

def get_url_from_entry ( entry ) : if 'url' in entry . fields : return entry . fields [ 'url' ] elif entry . type . lower ( ) == 'docushare' : return 'https://ls.st/' + entry . fields [ 'handle' ] elif 'adsurl' in entry . fields : return entry . fields [ 'adsurl' ] elif 'doi' in entry . fields : return 'https://doi.org/' + entry . fields [ 'doi' ] else : raise NoEntryUrlError ( )


def get_authoryear_from_entry(entry, paren=False):
    """Get and format author-year text from a pybtex entry to emulate
    natbib citations.

    Parameters
    ----------
    entry : `pybtex.database.Entry`
        A pybtex bibliography entry.
    parens : `bool`, optional
        Whether to add parentheses around the year. Default is `False`.

    Returns
    -------
    authoryear : `str`
        The author-year citation text.
    """
    def _format_last(person):
        """Reformat a pybtex Person into a last name.

        Joins all parts of a last name and strips "{}" wrappers.
        """
        return ' '.join([n.strip('{}') for n in person.last_names])

    if len(entry.persons['author']) > 0:
        # Grab author list
        persons = entry.persons['author']
    elif len(entry.persons['editor']) > 0:
        # Grab editor list
        persons = entry.persons['editor']
    else:
        raise AuthorYearError

    try:
        year = entry.fields['year']
    except KeyError:
        raise AuthorYearError

    if paren and len(persons) == 1:
        template = '{author} ({year})'
        return template.format(author=_format_last(persons[0]),
                               year=year)
    elif not paren and len(persons) == 1:
        template = '{author} {year}'
        return template.format(author=_format_last(persons[0]),
                               year=year)
    elif paren and len(persons) == 2:
        template = '{author1} and {author2} ({year})'
        return template.format(author1=_format_last(persons[0]),
                               author2=_format_last(persons[1]),
                               year=year)
    elif not paren and len(persons) == 2:
        template = '{author1} and {author2} {year}'
        return template.format(author1=_format_last(persons[0]),
                               author2=_format_last(persons[1]),
                               year=year)
    elif not paren and len(persons) > 2:
        template = '{author} et al {year}'
        return template.format(author=_format_last(persons[0]),
                               year=year)
    elif paren and len(persons) > 2:
        template = '{author} et al ({year})'
        return template.format(author=_format_last(persons[0]),
                               year=year)

Get and format author - year text from a pybtex entry to emulate natbib citations .


async def process_sphinx_technote(session, github_api_token, ltd_product_data,
                                  mongo_collection=None):
    """Extract, transform, and load Sphinx-based technote metadata.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    ltd_product_data : `dict`
        Contents of ``metadata.yaml``, obtained via `download_metadata_yaml`.
        Data for this technote from the LTD Keeper API
        (``GET /products/<slug>``). Usually obtained via
        `lsstprojectmeta.ltd.get_ltd_product`.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records. If provided, ths JSON-LD is upserted
        into the MongoDB collection.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    Raises
    ------
    NotSphinxTechnoteError
        Raised when the LTD product cannot be interpreted as a Sphinx-based
        technote project because it's missing a metadata.yaml file in its
        GitHub repository. This implies that the LTD product *could* be of a
        different format.

    .. `GitHub personal access token guide`: https://ls.st/41d
    """
    logger = logging.getLogger(__name__)

    github_url = ltd_product_data['doc_repo']
    github_url = normalize_repo_root_url(github_url)
    repo_slug = parse_repo_slug_from_url(github_url)

    try:
        metadata_yaml = await download_metadata_yaml(session, github_url)
    except aiohttp.ClientResponseError as err:
        # metadata.yaml not found; probably not a Sphinx technote
        logger.debug('Tried to download %s\'s metadata.yaml, got status %d',
                     ltd_product_data['slug'], err.code)
        raise NotSphinxTechnoteError()

    # Extract data from the GitHub API
    github_query = GitHubQuery.load('technote_repo')
    github_variables = {
        "orgName": repo_slug.owner,
        "repoName": repo_slug.repo
    }
    github_data = await github_request(session, github_api_token,
                                       query=github_query,
                                       variables=github_variables)

    try:
        jsonld = reduce_technote_metadata(
            github_url, metadata_yaml, github_data, ltd_product_data)
    except Exception as exception:
        message = "Issue building JSON-LD for technote %s"
        logger.exception(message, github_url, exception)
        raise

    if mongo_collection is not None:
        await _upload_to_mongodb(mongo_collection, jsonld)

    logger.info('Ingested technote %s into MongoDB', github_url)

    return jsonld

Extract transform and load Sphinx - based technote metadata .


def reduce_technote_metadata(github_url, metadata, github_data,
                             ltd_product_data):
    """Reduce a technote project's metadata from multiple sources into a
    single JSON-LD resource.

    Parameters
    ----------
    github_url : `str`
        URL of the technote's GitHub repository.
    metadata : `dict`
        The parsed contents of ``metadata.yaml`` found in a technote's
        repository.
    github_data : `dict`
        The contents of the ``technote_repo`` GitHub GraphQL API query.
    ltd_product_data : `dict`
        JSON dataset for the technote corresponding to the
        ``/products/<product>`` of LTD Keeper.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    .. `GitHub personal access token guide`: https://ls.st/41d
    """
    repo_slug = parse_repo_slug_from_url(github_url)

    # Initialize a schema.org/Report and schema.org/SoftwareSourceCode
    # linked data resource
    jsonld = {
        '@context': [
            "https://raw.githubusercontent.com/codemeta/codemeta/2.0-rc/"
            "codemeta.jsonld",
            "http://schema.org"],
        '@type': ['Report', 'SoftwareSourceCode'],
        'codeRepository': github_url
    }

    if 'url' in metadata:
        url = metadata['url']
    elif 'published_url' in ltd_product_data:
        url = ltd_product_data['published_url']
    else:
        raise RuntimeError('No identifying url could be found: '
                           '{}'.format(github_url))
    jsonld['@id'] = url
    jsonld['url'] = url

    if 'series' in metadata and 'serial_number' in metadata:
        jsonld['reportNumber'] = '{series}-{serial_number}'.format(**metadata)
    else:
        raise RuntimeError('No reportNumber: {}'.format(github_url))

    if 'doc_title' in metadata:
        jsonld['name'] = metadata['doc_title']

    if 'description' in metadata:
        jsonld['description'] = metadata['description']

    if 'authors' in metadata:
        jsonld['author'] = [{'@type': 'Person', 'name': author_name}
                            for author_name in metadata['authors']]

    if 'last_revised' in metadata:
        # Prefer getting the 'last_revised' date from metadata.yaml
        # since it's considered an override.
        jsonld['dateModified'] = datetime.datetime.strptime(
            metadata['last_revised'],
            '%Y-%m-%d')
    else:
        # Fallback to parsing the date of the last commit to the
        # default branch on GitHub (usually `master`).
        try:
            _repo_data = github_data['data']['repository']
            _master_data = _repo_data['defaultBranchRef']
            jsonld['dateModified'] = datetime.datetime.strptime(
                _master_data['target']['committedDate'],
                '%Y-%m-%dT%H:%M:%SZ')
        except KeyError:
            pass

    try:
        _license_data = github_data['data']['repository']['licenseInfo']
        _spdxId = _license_data['spdxId']
        if _spdxId is not None:
            _spdx_url = 'https://spdx.org/licenses/{}.html'.format(_spdxId)
            jsonld['license'] = _spdx_url
    except KeyError:
        pass

    try:
        # Find the README(|.md|.rst|*) file in the repo root
        _master_data = github_data['data']['repository']['defaultBranchRef']
        _files = _master_data['target']['tree']['entries']
        for _node in _files:
            filename = _node['name']
            normalized_filename = filename.lower()
            if normalized_filename.startswith('readme'):
                readme_url = make_raw_content_url(repo_slug, 'master',
                                                  filename)
                jsonld['readme'] = readme_url
                break
    except KeyError:
        pass

    # Assume Travis is the CI service (always true at the moment)
    travis_url = 'https://travis-ci.org/{}'.format(repo_slug.full)
    jsonld['contIntegration'] = travis_url

    return jsonld

Reduce a technote project s metadata from multiple sources into a single JSON - LD resource .


async def download_metadata_yaml(session, github_url):
    """Download the metadata.yaml file from a technote's GitHub repository.
    """
    metadata_yaml_url = _build_metadata_yaml_url(github_url)
    async with session.get(metadata_yaml_url) as response:
        response.raise_for_status()
        yaml_data = await response.text()
    return yaml.safe_load(yaml_data)

async def download_metadata_yaml ( session , github_url ) : metadata_yaml_url = _build_metadata_yaml_url ( github_url ) async with session . get ( metadata_yaml_url ) as response : response . raise_for_status ( ) yaml_data = await response . text ( ) return yaml . safe_load ( yaml_data )

Download the metadata . yaml file from a technote s GitHub repository .


def tz(self):
        """Return the timezone. If none is set use system timezone"""
        if not self._tz:
            self._tz = tzlocal.get_localzone().zone
        return self._tz

def tz ( self ) : if not self . _tz : self . _tz = tzlocal . get_localzone ( ) . zone return self . _tz

Return the timezone . If none is set use system timezone


def time(self, t):
        """Convert any timestamp into a datetime and save as _time"""
        _time = arrow.get(t).format('YYYY-MM-DDTHH:mm:ss')
        self._time = datetime.datetime.strptime(_time, '%Y-%m-%dT%H:%M:%S')

def time ( self , t ) : _time = arrow . get ( t ) . format ( 'YYYY-MM-DDTHH:mm:ss' ) self . _time = datetime . datetime . strptime ( _time , '%Y-%m-%dT%H:%M:%S' )


def as_dict(self):
        """Return a dict that represents the DayOneEntry"""
        entry_dict = {}
        entry_dict['UUID'] = self.uuid
        entry_dict['Creation Date'] = self.time
        entry_dict['Time Zone'] = self.tz
        if self.tags:
            entry_dict['Tags'] = self.tags
        entry_dict['Entry Text'] = self.text
        entry_dict['Starred'] = self.starred
        entry_dict['Location'] = self.location
        return entry_dict

def
as_dict
(
self
)
:
entry_dict
=
{
}
entry_dict
[
'UUID'
]
=
self
.
uuid
entry_dict
[
'Creation Date'
]
=
self
.
time
entry_dict
[
'Time Zone'
]
=
self
.
tz
if
self
.
tags
:
entry_dict
[
'Tags'
]
=
self
.
tags
entry_dict
[
'Entry Text'
]
=
self
.
text
entry_dict
[
'Starred'
]
=
self
.
starred
entry_dict
[
'Location'
]
=
self
.
location
return
entry_dict


def save(self, entry, with_location=True, debug=False):
        """Saves a DayOneEntry as a plist"""
        entry_dict = {}
        if isinstance(entry, DayOneEntry):
            # Get a dict of the DayOneEntry
            entry_dict = entry.as_dict()
        else:
            entry_dict = entry

        # Set the UUID
        entry_dict['UUID'] = uuid.uuid4().get_hex()
        if with_location and not entry_dict['Location']:
            entry_dict['Location'] = self.get_location()

        # Do we have everything needed?
        if not all ((entry_dict['UUID'], entry_dict['Time Zone'],
                     entry_dict['Entry Text'])):
            print "You must provide: Time zone, UUID, Creation Date, Entry Text"
            return False

        if debug is False:
            file_path = self._file_path(entry_dict['UUID'])
            plistlib.writePlist(entry_dict, file_path)
        else:
            plist = plistlib.writePlistToString(entry_dict)
            print plist

        return True


def _file_path(self, uid):
        """Create and return full file path for DayOne entry"""
        file_name = '%s.doentry' % (uid)
        return os.path.join(self.dayone_journal_path, file_name)

def _file_path ( self , uid ) : file_name = '%s.doentry' % ( uid ) return os . path . join ( self . dayone_journal_path , file_name )


def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None):
        """ Combine many files into a single file on disk.  Defaults to using the 'time' dimension. """
        nco = None
        try:
            nco = Nco()
        except BaseException:
            # This is not necessarily an import error (could be wrong PATH)
            raise ImportError("NCO not found.  The NCO python bindings are required to use 'Collection.combine'.")

        if len(members) > 0 and hasattr(members[0], 'path'):
            # A member DotDoct was passed in, we only need the paths
            members = [ m.path for m in members ]

        options  = ['-4']  # NetCDF4
        options += ['-L', '3']  # Level 3 compression
        options += ['-h']  # Don't append to the history global attribute
        if dimension is not None:
            if start_index is None:
                start_index = 0
            if stop_index is None:
                stop_index = ''
            if stride is None:
                stride = 1
            options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)]
        nco.ncrcat(input=members, output=output_file, options=options)

Combine many files into a single file on disk . Defaults to using the time dimension .


def main(argv=None, white_list=None, load_yaz_extension=True):
    """The entry point for a yaz script

    This will almost always be called from a python script in
    the following manner:

        if __name__ == "__main__":
            yaz.main()

    This function will perform the following steps:

    1. It will load any additional python code from
       the yaz_extension python module located in the
       ~/.yaz directory when LOAD_YAZ_EXTENSION is True
       and the yaz_extension module exists

    2. It collects all yaz tasks and plugins.  When WHITE_LIST
       is a non-empty list, only the tasks and plugins located
       therein will be considered

    3. It will parse arguments from ARGV, or the command line
       when ARGV is not given, resulting in a yaz task or a parser
       help message.

    4. When a suitable task is found, this task is executed.  In
       case of a task which is part of a plugin, i.e. class, then
       this plugin is initialized, possibly resulting in other
       plugins to also be initialized if there are marked as
       `@yaz.dependency`.
    """
    assert argv is None or isinstance(argv, list), type(argv)
    assert white_list is None or isinstance(white_list, list), type(white_list)
    assert isinstance(load_yaz_extension, bool), type(load_yaz_extension)

    argv = sys.argv if argv is None else argv
    assert len(argv) > 0, len(argv)

    if load_yaz_extension:
        load("~/.yaz", "yaz_extension")

    parser = Parser(prog=argv[0])
    parser.add_task_tree(get_task_tree(white_list))

    task, kwargs = parser.parse_arguments(argv)

    if task:
        try:
            result = task(**kwargs)

            # when the result is a boolean, exit with 0 (success) or 1 (failure)
            if isinstance(result, bool):
                code = 0 if result else 1
                output = None

            # when the result is an integer, exit with that integer value
            elif isinstance(result, int):
                code = result % 256
                output = None

            # otherwise exit with 0 (success) and print the result
            else:
                code = 0
                output = result

        # when yaz.Error occurs, exit with the given return code and print the error message
        # when any other error occurs, let python handle the exception (i.e. exit(1) and print call stack)
        except Error as error:
            code = error.return_code
            output = error

    else:
        # when no task is found to execute, exit with 1 (failure) and print the help text
        code = 1
        output = parser.format_help().rstrip()

    if output is not None:
        print(output)

    sys.exit(code)


def get_task_tree(white_list=None):
    """Returns a tree of Task instances

    The tree is comprised of dictionaries containing strings for
    keys and either dictionaries or Task instances for values.

    When WHITE_LIST is given, only the tasks and plugins in this
    list will become part of the task tree.  The WHITE_LIST may
    contain either strings, corresponding to the task of plugin
    __qualname__, or, preferable, the WHITE_LIST contains
    links to the task function or plugin class instead.
    """
    assert white_list is None or isinstance(white_list, list), type(white_list)

    if white_list is not None:
        white_list = set(item if isinstance(item, str) else item.__qualname__ for item in white_list)

    tree = dict((task.qualified_name, task)
                for task
                in _task_list.values()
                if white_list is None or task.qualified_name in white_list)

    plugins = get_plugin_list()
    for plugin in [plugin for plugin in plugins.values() if white_list is None or plugin.__qualname__ in white_list]:
        tasks = [func
                 for _, func
                 in inspect.getmembers(plugin)
                 if inspect.isfunction(func) and hasattr(func, "yaz_task_config")]
        if len(tasks) == 0:
            continue

        node = tree
        for name in plugin.__qualname__.split("."):
            if not name in node:
                node[name] = {}
            node = node[name]

        for func in tasks:
            logger.debug("Found task %s", func)
            node[func.__name__] = Task(plugin_class=plugin, func=func, config=func.yaz_task_config)

    return tree

def
get_task_tree
(
white_list
=
)
:
assert
white_list
is
or
isinstance
(
white_list
,
list
)
,
type
(
white_list
)
if
white_list
is
not
:
white_list
=
set
(
item
if
isinstance
(
item
,
str
)
else
item
.
__qualname__
for
item
in
white_list
)
tree
=
dict
(
(
task
.
qualified_name
,
task
)
for
task
in
_task_list
.
values
(
)
if
white_list
is
or
task
.
qualified_name
in
white_list
)
plugins
=
get_plugin_list
(
)
for
plugin
in
[
plugin
for
plugin
in
plugins
.
values
(
)
if
white_list
is
or
plugin
.
__qualname__
in
white_list
]
:
tasks
=
[
func
for
_
,
func
in
inspect
.
getmembers
(
plugin
)
if
inspect
.
isfunction
(
func
)
and
hasattr
(
func
,
"yaz_task_config"
)
]
if
len
(
tasks
)
==
0
:
continue
node
=
tree
for
name
in
plugin
.
__qualname__
.
split
(
"."
)
:
if
not
name
in
node
:
node
[
name
]
=
{
}
node
=
node
[
name
]
for
func
in
tasks
:
logger
.
debug
(
"Found task %s"
,
func
)
node
[
func
.
__name__
]
=
Task
(
plugin_class
=
plugin
,
func
=
func
,
config
=
func
.
yaz_task_config
)
return
tree


def task(func, **config):
    """Declare a function or method to be a Yaz task

    @yaz.task
    def talk(message: str = "Hello World!"):
        return message

    Or... group multiple tasks together

    class Tools(yaz.Plugin):
        @yaz.task
        def say(self, message: str = "Hello World!"):
            return message

        @yaz.task(option__choices=["A", "B", "C"])
        def choose(self, option: str = "A"):
            return option
    """
    if func.__name__ == func.__qualname__:
        assert not func.__qualname__ in _task_list, "Can not define the same task \"{}\" twice".format(func.__qualname__)
        logger.debug("Found task %s", func)
        _task_list[func.__qualname__] = Task(plugin_class=None, func=func, config=config)
    else:
        func.yaz_task_config = config

    return func


def get_parameters(self):
        """Returns a list of parameters"""
        if self.plugin_class is None:
            sig = inspect.signature(self.func)
            for index, parameter in enumerate(sig.parameters.values()):
                if not parameter.kind in [parameter.POSITIONAL_ONLY, parameter.KEYWORD_ONLY, parameter.POSITIONAL_OR_KEYWORD]:
                    raise RuntimeError("Task {} contains an unsupported {} parameter".format(parameter, parameter.kind))

                yield parameter

        else:
            var_keyword_seen = set()

            for cls in inspect.getmro(self.plugin_class):
                if issubclass(cls, BasePlugin) and hasattr(cls, self.func.__name__):
                    func = getattr(cls, self.func.__name__)
                    logger.debug("Found method %s from class %s", func, cls)
                    var_keyword_found = False
                    sig = inspect.signature(func)
                    for index, parameter in enumerate(sig.parameters.values()):
                        if index == 0:
                            # skip "self" parameter
                            continue

                        if parameter.kind == inspect.Parameter.VAR_KEYWORD:
                            # found "**kwargs" parameter.  we will continue to the next class in the mro
                            # to add any keyword parameters we have not yet used (i.e. whose name
                            # we have not yet seen)
                            var_keyword_found = True
                            continue

                        if parameter.kind in [parameter.POSITIONAL_ONLY, parameter.VAR_POSITIONAL]:
                            raise RuntimeError("Task {} contains an unsupported parameter \"{}\"".format(func, parameter))

                        if not parameter.name in var_keyword_seen:
                            var_keyword_seen.add(parameter.name)

                            logger.debug("Found parameter %s (%s)", parameter, parameter.kind)
                            yield parameter

                    # we only need to look at the next class in the mro
                    # when "**kwargs" is found
                    if not var_keyword_found:
                        break


def get_configuration(self, key, default=None):
        """Returns the configuration for KEY"""
        if key in self.config:
            return self.config.get(key)
        else:
            return default

def get_configuration ( self , key , default = None ) : if key in self . config : return self . config . get ( key ) else : return default


def get_plugin_instance(plugin_class, *args, **kwargs):
    """Returns an instance of a fully initialized plugin class

    Every plugin class is kept in a plugin cache, effectively making
    every plugin into a singleton object.

    When a plugin has a yaz.dependency decorator, it will be called
    as well, before the instance is returned.
    """
    assert issubclass(plugin_class, BasePlugin), type(plugin_class)

    global _yaz_plugin_instance_cache

    qualname = plugin_class.__qualname__
    if not qualname in _yaz_plugin_instance_cache:
        plugin_class = get_plugin_list()[qualname]
        _yaz_plugin_instance_cache[qualname] = plugin = plugin_class(*args, **kwargs)

        # find any yaz.dependency decorators, and call them when necessary
        funcs = [func
                 for _, func
                 in inspect.getmembers(plugin)
                 if inspect.ismethod(func) and hasattr(func, "yaz_dependency_config")]

        for func in funcs:
            signature = inspect.signature(func)
            assert all(parameter.kind is parameter.POSITIONAL_OR_KEYWORD and issubclass(parameter.annotation, BasePlugin) for parameter in signature.parameters.values()), "All parameters for {} must type hint to a BasePlugin".format(func)
            func(*[get_plugin_instance(parameter.annotation)
                   for parameter
                   in signature.parameters.values()])

    return _yaz_plugin_instance_cache[qualname]


def xml_to_json(root):
    """Convert an Open511 XML document or document fragment to JSON.

    Takes an lxml Element object. Returns a dict ready to be JSON-serialized."""
    j = {}

    if len(root) == 0:  # Tag with no children, return str/int
        return _maybe_intify(root.text)

    if len(root) == 1 and root[0].tag.startswith('{' + NS_GML):  # GML
        return gml_to_geojson(root[0])

    if root.tag == 'open511':
        j['meta'] = {'version': root.get('version')}

    for elem in root:
        name = elem.tag
        if name == 'link' and elem.get('rel'):
            name = elem.get('rel') + '_url'
            if name == 'self_url':
                name = 'url'
            if root.tag == 'open511':
                j['meta'][name] = elem.get('href')
                continue
        elif name.startswith('{' + NS_PROTECTED):
            name = '!' + name[name.index('}') + 1:]
        elif name[0] == '{':
            # Namespace!
            name = '+' + name[name.index('}') + 1:]

        if name in j:
            continue  # duplicate
        elif elem.tag == 'link' and not elem.text:
            j[name] = elem.get('href')
        elif len(elem):
            if name == 'grouped_events':
                # An array of URLs
                j[name] = [xml_link_to_json(child, to_dict=False) for child in elem]
            elif name in ('attachments', 'media_files'):
                # An array of JSON objects
                j[name] = [xml_link_to_json(child, to_dict=True) for child in elem]
            elif all((name == pluralize(child.tag) for child in elem)):
                # <something><somethings> serializes to a JSON array
                j[name] = [xml_to_json(child) for child in elem]
            else:
                j[name] = xml_to_json(elem)
        else:
            if root.tag == 'open511' and name.endswith('s') and not elem.text:
                # Special case: an empty e.g. <events /> container at the root level
                # should be serialized to [], not null
                j[name] = []
            else:
                j[name] = _maybe_intify(elem.text)

    return j

Convert an Open511 XML document or document fragment to JSON .


def gml_to_geojson(el):
    """Given an lxml Element of a GML geometry, returns a dict in GeoJSON format."""
    if el.get('srsName') not in ('urn:ogc:def:crs:EPSG::4326', None):
        if el.get('srsName') == 'EPSG:4326':
            return _gmlv2_to_geojson(el)
        else:
            raise NotImplementedError("Unrecognized srsName %s" % el.get('srsName'))
    tag = el.tag.replace('{%s}' % NS_GML, '')
    if tag == 'Point':
        coordinates = _reverse_gml_coords(el.findtext('{%s}pos' % NS_GML))[0]
    elif tag == 'LineString':
        coordinates = _reverse_gml_coords(el.findtext('{%s}posList' % NS_GML))
    elif tag == 'Polygon':
        coordinates = []
        for ring in el.xpath('gml:exterior/gml:LinearRing/gml:posList', namespaces=NSMAP) \
                + el.xpath('gml:interior/gml:LinearRing/gml:posList', namespaces=NSMAP):
            coordinates.append(_reverse_gml_coords(ring.text))
    elif tag in ('MultiPoint', 'MultiLineString', 'MultiPolygon'):
        single_type = tag[5:]
        member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        coordinates = [
            gml_to_geojson(member)['coordinates']
            for member in el.xpath('gml:%s/gml:%s' % (member_tag, single_type), namespaces=NSMAP)
        ]
    else:
        raise NotImplementedError

    return {
        'type': tag,
        'coordinates': coordinates
    }

Given an lxml Element of a GML geometry returns a dict in GeoJSON format .


def _gmlv2_to_geojson(el):
    """Translates a deprecated GML 2.0 geometry to GeoJSON"""
    tag = el.tag.replace('{%s}' % NS_GML, '')
    if tag == 'Point':
        coordinates = [float(c) for c in el.findtext('{%s}coordinates' % NS_GML).split(',')]
    elif tag == 'LineString':
        coordinates = [
            [float(x) for x in pair.split(',')]
            for pair in el.findtext('{%s}coordinates' % NS_GML).split(' ')
        ]
    elif tag == 'Polygon':
        coordinates = []
        for ring in el.xpath('gml:outerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP) \
                + el.xpath('gml:innerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP):
            coordinates.append([
                [float(x) for x in pair.split(',')]
                for pair in ring.text.split(' ')
            ])
    elif tag in ('MultiPoint', 'MultiLineString', 'MultiPolygon', 'MultiCurve'):
        if tag == 'MultiCurve':
            single_type = 'LineString'
            member_tag = 'curveMember'
        else:
            single_type = tag[5:]
            member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        coordinates = [
            gml_to_geojson(member)['coordinates']
            for member in el.xpath('gml:%s/gml:%s' % (member_tag, single_type), namespaces=NSMAP)
        ]
    else:
        raise NotImplementedError

    return {
        'type': tag,
        'coordinates': coordinates
    }

Translates a deprecated GML 2 . 0 geometry to GeoJSON


def deparagraph(element, doc):
    """Panflute filter function that converts content wrapped in a Para to
    Plain.

    Use this filter with pandoc as::

        pandoc [..] --filter=lsstprojectmeta-deparagraph

    Only lone paragraphs are affected. Para elements with siblings (like a
    second Para) are left unaffected.

    This filter is useful for processing strings like titles or author names so
    that the output isn't wrapped in paragraph tags. For example, without
    this filter, pandoc converts a string ``"The title"`` to
    ``
The title
`` in HTML. These ``
`` tags aren't useful if you
    intend to put the title text in ``
`` tags using your own templating
    system.
    """
    if isinstance(element, Para):
        # Check if siblings exist; don't process the paragraph in that case.
        if element.next is not None:
            return element
        elif element.prev is not None:
            return element

        # Remove the Para wrapper from the lone paragraph.
        # `Plain` is a container that isn't rendered as a paragraph.
        return Plain(*element.content)

Panflute filter function that converts content wrapped in a Para to Plain .


def all_subclasses(cls):
    """ Recursively generate of all the subclasses of class cls. """
    for subclass in cls.__subclasses__():
        yield subclass
        for subc in all_subclasses(subclass):
            yield subc

def all_subclasses ( cls ) : for subclass in cls . __subclasses__ ( ) : yield subclass for subc in all_subclasses ( subclass ) : yield subc

Recursively generate of all the subclasses of class cls .


def unique_justseen(iterable, key=None):
    "List unique elements, preserving order. Remember only the element just seen."
    # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B
    # unique_justseen('ABBCcAD', str.lower) --> A B C A D
    try:
        # PY2 support
        from itertools import imap as map
    except ImportError:
        from builtins import map

    return map(next, map(operator.itemgetter(1), itertools.groupby(iterable, key)))

List unique elements preserving order . Remember only the element just seen .


def generic_masked(arr, attrs=None, minv=None, maxv=None, mask_nan=True):
    """
    Returns a masked array with anything outside of values masked.
    The minv and maxv parameters take precendence over any dict values.
    The valid_range attribute takes precendence over the valid_min and
    valid_max attributes.
    """
    attrs = attrs or {}

    if 'valid_min' in attrs:
        minv = safe_attribute_typing(arr.dtype, attrs['valid_min'])
    if 'valid_max' in attrs:
        maxv = safe_attribute_typing(arr.dtype, attrs['valid_max'])
    if 'valid_range' in attrs:
        vr = attrs['valid_range']
        minv = safe_attribute_typing(arr.dtype, vr[0])
        maxv = safe_attribute_typing(arr.dtype, vr[1])

    # Get the min/max of values that the hardware supports
    try:
        info = np.iinfo(arr.dtype)
    except ValueError:
        info = np.finfo(arr.dtype)

    minv = minv if minv is not None else info.min
    maxv = maxv if maxv is not None else info.max

    if mask_nan is True:
        arr = np.ma.fix_invalid(arr)

    return np.ma.masked_outside(
        arr,
        minv,
        maxv
    )

Returns a masked array with anything outside of values masked . The minv and maxv parameters take precendence over any dict values . The valid_range attribute takes precendence over the valid_min and valid_max attributes .


def default(self, obj):
        """If input object is an ndarray it will be converted into a list
        """
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, np.generic):
            return np.asscalar(obj)
        # Let the base class default method raise the TypeError
        return json.JSONEncoder(self, obj)


def default(self, obj):
        """If input object is an ndarray it will be converted into a dict
        holding dtype, shape and the data, base64 encoded.
        """
        if isinstance(obj, np.ndarray):
            if obj.flags['C_CONTIGUOUS']:
                obj_data = obj.data
            else:
                cont_obj = np.ascontiguousarray(obj)
                assert(cont_obj.flags['C_CONTIGUOUS'])
                obj_data = cont_obj.data
            data_b64 = base64.b64encode(obj_data)
            return dict(__ndarray__=data_b64,
                        dtype=str(obj.dtype),
                        shape=obj.shape)
        elif isinstance(obj, np.generic):
            return np.asscalar(obj)
        # Let the base class default method raise the TypeError
        return json.JSONEncoder(self, obj)

If input object is an ndarray it will be converted into a dict holding dtype shape and the data base64 encoded .


def update_desc_lsib_path(desc):
    '''
        leftSibling
        previousSibling
        leftSib
        prevSib
        lsib
        psib

        have the same parent,and on the left
    '''
    if(desc['sib_seq']>0):
        lsib_path = copy.deepcopy(desc['path'])
        lsib_path[-1] = desc['sib_seq']-1
        desc['lsib_path'] = lsib_path
    else:
        pass
    return(desc)

def update_desc_lsib_path ( desc ) : if ( desc [ 'sib_seq' ] > 0 ) : lsib_path = copy . deepcopy ( desc [ 'path' ] ) lsib_path [ - 1 ] = desc [ 'sib_seq' ] - 1 desc [ 'lsib_path' ] = lsib_path else : pass return ( desc )

leftSibling previousSibling leftSib prevSib lsib psib have the same parent and on the left


def update_desc_rsib_path(desc,sibs_len):
    '''
        rightSibling
        nextSibling
        rightSib
        nextSib
        rsib
        nsib

        have the same parent,and on the right
    '''
    if(desc['sib_seq']<(sibs_len-1)):
        rsib_path = copy.deepcopy(desc['path'])
        rsib_path[-1] = desc['sib_seq']+1
        desc['rsib_path'] = rsib_path
    else:
        pass
    return(desc)

def update_desc_rsib_path ( desc , sibs_len ) : if ( desc [ 'sib_seq' ] < ( sibs_len - 1 ) ) : rsib_path = copy . deepcopy ( desc [ 'path' ] ) rsib_path [ - 1 ] = desc [ 'sib_seq' ] + 1 desc [ 'rsib_path' ] = rsib_path else : pass return ( desc )

rightSibling nextSibling rightSib nextSib rsib nsib have the same parent and on the right


def update_desc_lcin_path(desc,pdesc_level):
    '''
        leftCousin
        previousCousin
        leftCin
        prevCin
        lcin
        pcin

        parents are neighbors,and on the left
    '''
    parent_breadth = desc['parent_breadth_path'][-1]
    if(desc['sib_seq']==0):
        if(parent_breadth==0):
            pass
        else:
            parent_lsib_breadth = parent_breadth - 1
            plsib_desc = pdesc_level[parent_lsib_breadth]
            if(plsib_desc['leaf']):
                pass
            else:
                lcin_path = copy.deepcopy(plsib_desc['path'])
                lcin_path.append(plsib_desc['sons_count'] - 1)
                desc['lcin_path'] = lcin_path
    else:
        pass
    return(desc)

def update_desc_lcin_path ( desc , pdesc_level ) : parent_breadth = desc [ 'parent_breadth_path' ] [ - 1 ] if ( desc [ 'sib_seq' ] == 0 ) : if ( parent_breadth == 0 ) : pass else : parent_lsib_breadth = parent_breadth - 1 plsib_desc = pdesc_level [ parent_lsib_breadth ] if ( plsib_desc [ 'leaf' ] ) : pass else : lcin_path = copy . deepcopy ( plsib_desc [ 'path' ] ) lcin_path . append ( plsib_desc [ 'sons_count' ] - 1 ) desc [ 'lcin_path' ] = lcin_path else : pass return ( desc )

leftCousin previousCousin leftCin prevCin lcin pcin parents are neighbors and on the left


def update_desc_rcin_path(desc,sibs_len,pdesc_level):
    '''
        rightCousin
        nextCousin
        rightCin
        nextCin
        rcin
        ncin

        parents are neighbors,and on the right
    '''
    psibs_len = pdesc_level.__len__()
    parent_breadth = desc['parent_breadth_path'][-1]
    if(desc['sib_seq']==(sibs_len - 1)):
        if(parent_breadth==(psibs_len -1)):
            pass
        else:
            parent_rsib_breadth = parent_breadth + 1
            prsib_desc = pdesc_level[parent_rsib_breadth]
            #because from left to right to handle each level
            #sons_count will only be updated in the next-round
            if(prsib_desc['leaf']):
                pass
            else:
                rcin_path = copy.deepcopy(prsib_desc['path'])
                rcin_path.append(0)
                desc['rcin_path'] = rcin_path
    else:
        pass
    return(desc)

rightCousin nextCousin rightCin nextCin rcin ncin parents are neighbors and on the right


def child_begin_handler(self,scache,*args):
        '''
            _creat_child_desc
            update depth,parent_breadth_path,parent_path,sib_seq,path,lsib_path,rsib_path,lcin_path,rcin_path
        '''
        pdesc = self.pdesc
        depth = scache.depth
        sib_seq = self.sib_seq
        sibs_len = self.sibs_len
        pdesc_level = scache.pdesc_level
        desc = copy.deepcopy(pdesc)
        desc = reset_parent_desc_template(desc)
        desc['depth'] = depth
        desc['parent_breadth_path'] = copy.deepcopy(desc['breadth_path'])
        desc['sib_seq'] = sib_seq
        desc['parent_path'] = copy.deepcopy(desc['path'])
        desc['path'].append(sib_seq)
        update_desc_lsib_path(desc)
        update_desc_rsib_path(desc,sibs_len)
        if(depth == 1):
            pass
        else:
            update_desc_lcin_path(desc,pdesc_level)
            update_desc_rcin_path(desc,sibs_len,pdesc_level)
        return(desc)

def child_begin_handler ( self , scache , * args ) : pdesc = self . pdesc depth = scache . depth sib_seq = self . sib_seq sibs_len = self . sibs_len pdesc_level = scache . pdesc_level desc = copy . deepcopy ( pdesc ) desc = reset_parent_desc_template ( desc ) desc [ 'depth' ] = depth desc [ 'parent_breadth_path' ] = copy . deepcopy ( desc [ 'breadth_path' ] ) desc [ 'sib_seq' ] = sib_seq desc [ 'parent_path' ] = copy . deepcopy ( desc [ 'path' ] ) desc [ 'path' ] . append ( sib_seq ) update_desc_lsib_path ( desc ) update_desc_rsib_path ( desc , sibs_len ) if ( depth == 1 ) : pass else : update_desc_lcin_path ( desc , pdesc_level ) update_desc_rcin_path ( desc , sibs_len , pdesc_level ) return ( desc )


def child_end_handler(self,scache):
        '''
            _upgrade_breadth_info
            update breadth, breadth_path, and add desc to desc_level
        '''
        desc = self.desc
        desc_level = scache.desc_level
        breadth = desc_level.__len__()
        desc['breadth'] = breadth
        desc['breadth_path'].append(breadth)
        desc_level.append(desc)

def child_end_handler ( self , scache ) : desc = self . desc desc_level = scache . desc_level breadth = desc_level . __len__ ( ) desc [ 'breadth' ] = breadth desc [ 'breadth_path' ] . append ( breadth ) desc_level . append ( desc )

_upgrade_breadth_info update breadth breadth_path and add desc to desc_level


def parse(self, source):
        """Parse command content from the LaTeX source.

        Parameters
        ----------
        source : `str`
            The full source of the tex document.

        Yields
        ------
        parsed_command : `ParsedCommand`
            Yields parsed commands instances for each occurence of the command
            in the source.
        """
        command_regex = self._make_command_regex(self.name)
        for match in re.finditer(command_regex, source):
            self._logger.debug(match)
            start_index = match.start(0)
            yield self._parse_command(source, start_index)

def parse ( self , source ) : command_regex = self . _make_command_regex ( self . name ) for match in re . finditer ( command_regex , source ) : self . _logger . debug ( match ) start_index = match . start ( 0 ) yield self . _parse_command ( source , start_index )

Parse command content from the LaTeX source .


def _parse_command(self, source, start_index):
        """Parse a single command.

        Parameters
        ----------
        source : `str`
            The full source of the tex document.
        start_index : `int`
            Character index in ``source`` where the command begins.

        Returns
        -------
        parsed_command : `ParsedCommand`
            The parsed command from the source at the given index.
        """
        parsed_elements = []

        # Index of the parser in the source
        running_index = start_index

        for element in self.elements:
            opening_bracket = element['bracket']
            closing_bracket = self._brackets[opening_bracket]

            # Find the opening bracket.
            element_start = None
            element_end = None
            for i, c in enumerate(source[running_index:], start=running_index):
                if c == element['bracket']:
                    element_start = i
                    break
                elif c == '\n':
                    # No starting bracket on the line.
                    if element['required'] is True:
                        # Try to parse a single single-word token after the
                        # command, like '\input file'
                        content = self._parse_whitespace_argument(
                            source[running_index:],
                            self.name)
                        return ParsedCommand(
                            self.name,
                            [{'index': element['index'],
                              'name': element['name'],
                              'content': content.strip()}],
                            start_index,
                            source[start_index:i])
                    else:
                        # Give up on finding an optional element
                        break

            # Handle cases when the opening bracket is never found.
            if element_start is None and element['required'] is False:
                # Optional element not found. Continue to next element,
                # not advancing the running_index of the parser.
                continue
            elif element_start is None and element['required'] is True:
                message = ('Parsing command {0} at index {1:d}, '
                           'did not detect element {2:d}'.format(
                               self.name,
                               start_index,
                               element['index']))
                raise CommandParserError(message)

            # Find the closing bracket, keeping track of the number of times
            # the same type of bracket was opened and closed.
            balance = 1
            for i, c in enumerate(source[element_start + 1:],
                                  start=element_start + 1):
                if c == opening_bracket:
                    balance += 1
                elif c == closing_bracket:
                    balance -= 1

                if balance == 0:
                    element_end = i
                    break

            if balance > 0:
                message = ('Parsing command {0} at index {1:d}, '
                           'did not find closing bracket for required '
                           'command element {2:d}'.format(
                               self.name,
                               start_index,
                               element['index']))
                raise CommandParserError(message)

            # Package the parsed element's content.
            element_content = source[element_start + 1:element_end]
            parsed_element = {
                'index': element['index'],
                'name': element['name'],
                'content': element_content.strip()
            }
            parsed_elements.append(parsed_element)

            running_index = element_end + 1

        command_source = source[start_index:running_index]
        parsed_command = ParsedCommand(self.name, parsed_elements,
                                       start_index, command_source)
        return parsed_command


def _parse_whitespace_argument(source, name):
        r"""Attempt to parse a single token on the first line of this source.

        This method is used for parsing whitespace-delimited arguments, like
        ``\input file``. The source should ideally contain `` file`` along
        with a newline character.

        >>> source = 'Line 1\n' r'\input test.tex' '\nLine 2'
        >>> LatexCommand._parse_whitespace_argument(source, 'input')
        'test.tex'

        Bracket delimited arguments (``\input{test.tex}``) are handled in
        the normal logic of `_parse_command`.
        """
        # First match the command name itself so that we find the argument
        # *after* the command
        command_pattern = r'\\(' + name + r')(?:[\s{[%])'
        command_match = re.search(command_pattern, source)
        if command_match is not None:
            # Trim `source` so we only look after the command
            source = source[command_match.end(1):]

        # Find the whitespace-delimited argument itself.
        pattern = r'(?P<content>\S+)(?:[ %\t\n]+)'
        match = re.search(pattern, source)
        if match is None:
            message = (
                'When parsing {}, did not find whitespace-delimited command '
                'argument'
            )
            raise CommandParserError(message.format(name))
        content = match.group('content')
        content.strip()
        return content

r Attempt to parse a single token on the first line of this source .


def list_from_document(cls, doc):
        """Returns a list of TMDDEventConverter elements.

        doc is an XML Element containing one or more <FEU> events
        """
        objs = []
        for feu in doc.xpath('//FEU'):
            detail_els = feu.xpath('event-element-details/event-element-detail')
            for idx, detail in enumerate(detail_els):
                objs.append(cls(feu, detail, id_suffix=idx, number_in_group=len(detail_els)))
        return objs

Returns a list of TMDDEventConverter elements .


def clone(src, dst_path, skip_globals, skip_dimensions, skip_variables):
    """
        Mostly ripped from nc3tonc4 in netCDF4-python.
        Added ability to skip dimension and variables.
        Removed all of the unpacking logic for shorts.
    """

    if os.path.exists(dst_path):
        os.unlink(dst_path)
    dst = netCDF4.Dataset(dst_path, 'w')

    # Global attributes
    for attname in src.ncattrs():
        if attname not in skip_globals:
            setattr(dst, attname, getattr(src, attname))

    # Dimensions
    unlimdim     = None
    unlimdimname = False
    for dimname, dim in src.dimensions.items():

        # Skip what we need to
        if dimname in skip_dimensions:
            continue

        if dim.isunlimited():
            unlimdim     = dim
            unlimdimname = dimname
            dst.createDimension(dimname, None)
        else:
            dst.createDimension(dimname, len(dim))

    # Variables
    for varname, ncvar in src.variables.items():

        # Skip what we need to
        if varname in skip_variables:
            continue

        hasunlimdim = False
        if unlimdimname and unlimdimname in ncvar.dimensions:
            hasunlimdim = True

        filler = None
        if hasattr(ncvar, '_FillValue'):
            filler = ncvar._FillValue

        if ncvar.chunking == "contiguous":
            var = dst.createVariable(varname, ncvar.dtype, ncvar.dimensions, fill_value=filler)
        else:
            var = dst.createVariable(varname, ncvar.dtype, ncvar.dimensions, fill_value=filler, chunksizes=ncvar.chunking())

        # Attributes
        for attname in ncvar.ncattrs():
            if attname == '_FillValue':
                continue
            else:
                setattr(var, attname, getattr(ncvar, attname))

        # Data
        nchunk = 1000
        if hasunlimdim:
            if nchunk:
                start = 0
                stop = len(unlimdim)
                step = nchunk
                if step < 1:
                    step = 1
                for n in range(start, stop, step):
                    nmax = n + nchunk
                    if nmax > len(unlimdim):
                        nmax = len(unlimdim)
                    idata = ncvar[n:nmax]
                    var[n:nmax] = idata
            else:
                idata = ncvar[:]
                var[0:len(unlimdim)] = idata
        else:
            idata = ncvar[:]
            var[:] = idata

        dst.sync()

    src.close()
    dst.close()

def
clone
(
src
,
dst_path
,
skip_globals
,
skip_dimensions
,
skip_variables
)
:
if
os
.
path
.
exists
(
dst_path
)
:
os
.
unlink
(
dst_path
)
dst
=
netCDF4
.
Dataset
(
dst_path
,
'w'
)
# Global attributes
for
attname
in
src
.
ncattrs
(
)
:
if
attname
not
in
skip_globals
:
setattr
(
dst
,
attname
,
getattr
(
src
,
attname
)
)
# Dimensions
unlimdim
=
unlimdimname
=
False
for
dimname
,
dim
in
src
.
dimensions
.
items
(
)
:
# Skip what we need to
if
dimname
in
skip_dimensions
:
continue
if
dim
.
isunlimited
(
)
:
unlimdim
=
dim
unlimdimname
=
dimname
dst
.
createDimension
(
dimname
,
)
else
:
dst
.
createDimension
(
dimname
,
len
(
dim
)
)
# Variables
for
varname
,
ncvar
in
src
.
variables
.
items
(
)
:
# Skip what we need to
if
varname
in
skip_variables
:
continue
hasunlimdim
=
False
if
unlimdimname
and
unlimdimname
in
ncvar
.
dimensions
:
hasunlimdim
=
True
filler
=
if
hasattr
(
ncvar
,
'_FillValue'
)
:
filler
=
ncvar
.
_FillValue
if
ncvar
.
chunking
==
"contiguous"
:
var
=
dst
.
createVariable
(
varname
,
ncvar
.
dtype
,
ncvar
.
dimensions
,
fill_value
=
filler
)
else
:
var
=
dst
.
createVariable
(
varname
,
ncvar
.
dtype
,
ncvar
.
dimensions
,
fill_value
=
filler
,
chunksizes
=
ncvar
.
chunking
(
)
)
# Attributes
for
attname
in
ncvar
.
ncattrs
(
)
:
if
attname
==
'_FillValue'
:
continue
else
:
setattr
(
var
,
attname
,
getattr
(
ncvar
,
attname
)
)
# Data
nchunk
=
1000
if
hasunlimdim
:
if
nchunk
:
start
=
0
stop
=
len
(
unlimdim
)
step
=
nchunk
if
step
<
1
:
step
=
1
for
n
in
range
(
start
,
stop
,
step
)
:
nmax
=
n
+
nchunk
if
nmax
>
len
(
unlimdim
)
:
nmax
=
len
(
unlimdim
)
idata
=
ncvar
[
n
:
nmax
]
var
[
n
:
nmax
]
=
idata
else
:
idata
=
ncvar
[
:
]
var
[
0
:
len
(
unlimdim
)
]
=
idata
else
:
idata
=
ncvar
[
:
]
var
[
:
]
=
idata
dst
.
sync
(
)
src
.
close
(
)
dst
.
close
(
)

Mostly ripped from nc3tonc4 in netCDF4 - python . Added ability to skip dimension and variables . Removed all of the unpacking logic for shorts .


def get_dataframe_from_variable(nc, data_var):
    """ Returns a Pandas DataFrame of the data.
        This always returns positive down depths
    """
    time_var = nc.get_variables_by_attributes(standard_name='time')[0]

    depth_vars = nc.get_variables_by_attributes(axis=lambda v: v is not None and v.lower() == 'z')
    depth_vars += nc.get_variables_by_attributes(standard_name=lambda v: v in ['height', 'depth' 'surface_altitude'], positive=lambda x: x is not None)

    # Find the correct depth variable
    depth_var = None
    for d in depth_vars:
        try:
            if d._name in data_var.coordinates.split(" ") or d._name in data_var.dimensions:
                depth_var = d
                break
        except AttributeError:
            continue

    times  = netCDF4.num2date(time_var[:], units=time_var.units, calendar=getattr(time_var, 'calendar', 'standard'))
    original_times_size = times.size

    if depth_var is None and hasattr(data_var, 'sensor_depth'):
        depth_type = get_type(data_var.sensor_depth)
        depths = np.asarray([data_var.sensor_depth] * len(times)).flatten()
        values = data_var[:].flatten()
    elif depth_var is None:
        depths = np.asarray([np.nan] * len(times)).flatten()
        depth_type = get_type(depths)
        values = data_var[:].flatten()
    else:
        depths = depth_var[:]
        depth_type = get_type(depths)
        if len(data_var.shape) > 1:
            times = np.repeat(times, depths.size)
            depths = np.tile(depths, original_times_size)
            values = data_var[:, :].flatten()
        else:
            values = data_var[:].flatten()

        if getattr(depth_var, 'positive', 'down').lower() == 'up':
            logger.warning("Converting depths to positive down before returning the DataFrame")
            depths = depths * -1

    # https://github.com/numpy/numpy/issues/4595
    # We can't call astype on a MaskedConstant
    if (
        isinstance(depths, np.ma.core.MaskedConstant) or
        (hasattr(depths, 'mask') and depths.mask.all())
    ):
        depths = np.asarray([np.nan] * len(times)).flatten()

    df = pd.DataFrame({ 'time':   times,
                        'value':  values.astype(data_var.dtype),
                        'unit':   data_var.units if hasattr(data_var, 'units') else np.nan,
                        'depth':  depths.astype(depth_type) })

    df.set_index([pd.DatetimeIndex(df['time']), pd.Float64Index(df['depth'])], inplace=True)
    return df

Returns a Pandas DataFrame of the data . This always returns positive down depths


def load(cls, query_name):
        """Load a pre-made query.

        These queries are distributed with lsstprojectmeta. See
        :file:`lsstrojectmeta/data/githubv4/README.rst` inside the
        package repository for details on available queries.

        Parameters
        ----------
        query_name : `str`
            Name of the query, such as ``'technote_repo'``.

        Returns
        -------
        github_query : `GitHubQuery
            A GitHub query or mutation object that you can pass to
            `github_request` to execute the request itself.
        """
        template_path = os.path.join(
            os.path.dirname(__file__),
            '../data/githubv4',
            query_name + '.graphql')

        with open(template_path) as f:
            query_data = f.read()

        return cls(query_data, name=query_name)

def load ( cls , query_name ) : template_path = os . path . join ( os . path . dirname ( __file__ ) , '../data/githubv4' , query_name + '.graphql' ) with open ( template_path ) as f : query_data = f . read ( ) return cls ( query_data , name = query_name )


def read_git_commit_timestamp_for_file(filepath, repo_path=None, repo=None):
    """Obtain the timestamp for the most recent commit to a given file in a
    Git repository.

    Parameters
    ----------
    filepath : `str`
        Absolute or repository-relative path for a file.
    repo_path : `str`, optional
        Path to the Git repository. Leave as `None` to use the current working
        directory or if a ``repo`` argument is provided.
    repo : `git.Repo`, optional
        A `git.Repo` instance.

    Returns
    -------
    commit_timestamp : `datetime.datetime`
        The datetime of the most recent commit to the given file.

    Raises
    ------
    IOError
        Raised if the ``filepath`` does not exist in the Git repository.
    """
    logger = logging.getLogger(__name__)

    if repo is None:
        repo = git.repo.base.Repo(path=repo_path,
                                  search_parent_directories=True)
    repo_path = repo.working_tree_dir

    head_commit = repo.head.commit

    # filepath relative to the repo path
    logger.debug('Using Git repo at %r', repo_path)
    filepath = os.path.relpath(
        os.path.abspath(filepath),
        start=repo_path)
    logger.debug('Repo-relative filepath is %r', filepath)

    # Most recent commit datetime of the given file.
    # Don't use head_commit.iter_parents because then it skips the
    # commit of a file that's added but never modified.
    for commit in head_commit.iter_items(repo,
                                         head_commit,
                                         [filepath],
                                         skip=0):
        return commit.committed_datetime

    # Only get here if git could not find the file path in the history
    raise IOError('File {} not found'.format(filepath))

Obtain the timestamp for the most recent commit to a given file in a Git repository .


def get_content_commit_date(extensions, acceptance_callback=None,
                            root_dir='.'):
    """Get the datetime for the most recent commit to a project that
    affected certain types of content.

    Parameters
    ----------
    extensions : sequence of 'str'
        Extensions of files to consider in getting the most recent commit
        date. For example, ``('rst', 'svg', 'png')`` are content extensions
        for a Sphinx project. **Extension comparision is case sensitive.** add
        uppercase variants to match uppercase extensions.
    acceptance_callback : callable
        Callable function whose sole argument is a file path, and returns
        `True` or `False` depending on whether the file's commit date should
        be considered or not. This callback is only run on files that are
        included by ``extensions``. Thus this callback is a way to exclude
        specific files that would otherwise be included by their extension.
    root_dir : 'str`, optional
        Only content contained within this root directory is considered.
        This directory must be, or be contained by, a Git repository. This is
        the current working directory by default.

    Returns
    -------
    commit_date : `datetime.datetime`
        Datetime of the most recent content commit.

    Raises
    ------
    RuntimeError
        Raised if no content files are found.
    """
    logger = logging.getLogger(__name__)

    def _null_callback(_):
        return True

    if acceptance_callback is None:
        acceptance_callback = _null_callback

    # Cache the repo object for each query
    root_dir = os.path.abspath(root_dir)
    repo = git.repo.base.Repo(path=root_dir, search_parent_directories=True)

    # Iterate over all files with all file extensions, looking for the
    # newest commit datetime.
    newest_datetime = None
    iters = [_iter_filepaths_with_extension(ext, root_dir=root_dir)
             for ext in extensions]
    for content_path in itertools.chain(*iters):
        content_path = os.path.abspath(os.path.join(root_dir, content_path))

        if acceptance_callback(content_path):
            logger.debug('Found content path %r', content_path)
            try:
                commit_datetime = read_git_commit_timestamp_for_file(
                    content_path, repo=repo)
                logger.debug('Commit timestamp of %r is %s',
                             content_path, commit_datetime)
            except IOError:
                logger.warning(
                    'Count not get commit for %r, skipping',
                    content_path)
                continue

            if not newest_datetime or commit_datetime > newest_datetime:
                # Seed initial newest_datetime
                # or set a newer newest_datetime
                newest_datetime = commit_datetime
                logger.debug('Newest commit timestamp is %s', newest_datetime)

        logger.debug('Final commit timestamp is %s', newest_datetime)

    if newest_datetime is None:
        raise RuntimeError('No content files found in {}'.format(root_dir))

    return newest_datetime

Get the datetime for the most recent commit to a project that affected certain types of content .


def _iter_filepaths_with_extension(extname, root_dir='.'):
    """Iterative over relative filepaths of files in a directory, and
    sub-directories, with the given extension.

    Parameters
    ----------
    extname : `str`
        Extension name (such as 'txt' or 'rst'). Extension comparison is
        case sensitive.
    root_dir : 'str`, optional
        Root directory. Current working directory by default.

    Yields
    ------
    filepath : `str`
        File path, relative to ``root_dir``, with the given extension.
    """
    # needed for comparison with os.path.splitext
    if not extname.startswith('.'):
        extname = '.' + extname

    root_dir = os.path.abspath(root_dir)

    for dirname, sub_dirnames, filenames in os.walk(root_dir):
        for filename in filenames:
            if os.path.splitext(filename)[-1] == extname:
                full_filename = os.path.join(dirname, filename)
                rel_filepath = os.path.relpath(full_filename, start=root_dir)
                yield rel_filepath

Iterative over relative filepaths of files in a directory and sub - directories with the given extension .


def get_variables_by_attributes(self, **kwargs):
        """ Returns variables that match specific conditions.

        * Can pass in key=value parameters and variables are returned that
        contain all of the matches.  For example,

        >>> # Get variables with x-axis attribute.
        >>> vs = nc.get_variables_by_attributes(axis='X')
        >>> # Get variables with matching "standard_name" attribute.
        >>> nc.get_variables_by_attributes(standard_name='northward_sea_water_velocity')

        * Can pass in key=callable parameter and variables are returned if the
        callable returns True.  The callable should accept a single parameter,
        the attribute value.  None is given as the attribute value when the
        attribute does not exist on the variable. For example,

        >>> # Get Axis variables.
        >>> vs = nc.get_variables_by_attributes(axis=lambda v: v in ['X', 'Y', 'Z', 'T'])
        >>> # Get variables that don't have an "axis" attribute.
        >>> vs = nc.get_variables_by_attributes(axis=lambda v: v is None)
        >>> # Get variables that have a "grid_mapping" attribute.
        >>> vs = nc.get_variables_by_attributes(grid_mapping=lambda v: v is not None)

        """
        vs = []

        has_value_flag  = False
        for vname in self.variables:
            var = self.variables[vname]
            for k, v in kwargs.items():
                if callable(v):
                    has_value_flag = v(getattr(var, k, None))
                    if has_value_flag is False:
                        break
                elif hasattr(var, k) and getattr(var, k) == v:
                    has_value_flag = True
                else:
                    has_value_flag = False
                    break

            if has_value_flag is True:
                vs.append(self.variables[vname])

        return vs

def get_variables_by_attributes ( self , * * kwargs ) : vs = [ ] has_value_flag = False for vname in self . variables : var = self . variables [ vname ] for k , v in kwargs . items ( ) : if callable ( v ) : has_value_flag = v ( getattr ( var , k , None ) ) if has_value_flag is False : break elif hasattr ( var , k ) and getattr ( var , k ) == v : has_value_flag = True else : has_value_flag = False break if has_value_flag is True : vs . append ( self . variables [ vname ] ) return vs

Returns variables that match specific conditions .


def json_attributes(self, vfuncs=None):
        """
        vfuncs can be any callable that accepts a single argument, the
        Variable object, and returns a dictionary of new attributes to
        set. These will overwrite existing attributes
        """

        vfuncs = vfuncs or []

        js = {'global': {}}

        for k in self.ncattrs():
            js['global'][k] = self.getncattr(k)

        for varname, var in self.variables.items():
            js[varname] = {}
            for k in var.ncattrs():
                z = var.getncattr(k)
                try:
                    assert not np.isnan(z).all()
                    js[varname][k] = z
                except AssertionError:
                    js[varname][k] = None
                except TypeError:
                    js[varname][k] = z

            for vf in vfuncs:
                try:
                    js[varname].update(vfuncs(var))
                except BaseException:
                    logger.exception("Could not apply custom variable attribue function")

        return json.loads(json.dumps(js, cls=BasicNumpyEncoder))

vfuncs can be any callable that accepts a single argument the Variable object and returns a dictionary of new attributes to set . These will overwrite existing attributes


def ensure_pandoc(func):
    """Decorate a function that uses pypandoc to ensure that pandoc is
    installed if necessary.
    """
    logger = logging.getLogger(__name__)

    @functools.wraps(func)
    def _install_and_run(*args, **kwargs):
        try:
            # First try to run pypandoc function
            result = func(*args, **kwargs)
        except OSError:
            # Install pandoc and retry
            message = "pandoc needed but not found. Now installing it for you."
            logger.warning(message)
            # This version of pandoc is known to be compatible with both
            # pypandoc.download_pandoc and the functionality that
            # lsstprojectmeta needs. Travis CI tests are useful for ensuring
            # download_pandoc works.
            pypandoc.download_pandoc(version='1.19.1')
            logger.debug("pandoc download complete")

            result = func(*args, **kwargs)

        return result

    return _install_and_run

Decorate a function that uses pypandoc to ensure that pandoc is installed if necessary .