<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import abc <EOL> import logging <EOL> import os <EOL> import re <EOL> import shutil <EOL> import tarfile <EOL> import tempfile <EOL> from contextlib import closing <EOL> from contextlib import contextmanager <EOL> from rdflib import plugin <EOL> from rdflib . graph import Graph <EOL> from rdflib . store import Store <EOL> from rdflib . term import URIRef <EOL> from six import with_metaclass <EOL> from gutenberg . _domain_model . exceptions import CacheAlreadyExistsException <EOL> from gutenberg . _domain_model . exceptions import InvalidCacheException <EOL> from gutenberg . _domain_model . persistence import local_path <EOL> from gutenberg . _domain_model . vocabulary import DCTERMS <EOL> from gutenberg . _domain_model . vocabulary import PGTERMS <EOL> from gutenberg . _util . logging import disable_logging <EOL> from gutenberg . _util . os import makedirs <EOL> from gutenberg . _util . os import remove <EOL> from gutenberg . _util . url import urlopen <EOL> _GUTENBERG_CATALOG_URL = r'<STR_LIT>' <EOL> _DB_IDENTIFIER = '<STR_LIT>' <EOL> _DB_PATH = local_path ( os . path . join ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> class MetadataCache ( with_metaclass ( abc . ABCMeta , object ) ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , store , cache_uri ) : <EOL> self . store = store <EOL> self . cache_uri = cache_uri <EOL> self . graph = Graph ( store = self . store , identifier = _DB_IDENTIFIER ) <EOL> self . is_open = False <EOL> self . catalog_source = _GUTENBERG_CATALOG_URL <EOL> @ property <EOL> def exists ( self ) : <EOL> """<STR_LIT>""" <EOL> return os . path . exists ( self . _local_storage_path ) <EOL> def open ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . graph . open ( self . cache_uri , create = False ) <EOL> self . _add_namespaces ( self . graph ) <EOL> self . is_open = True <EOL> except : <EOL> raise InvalidCacheException ( '<STR_LIT>' ) <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> self . graph . close ( ) <EOL> self . is_open = False <EOL> def delete ( self ) : <EOL> """<STR_LIT>""" <EOL> self . close ( ) <EOL> remove ( self . _local_storage_path ) <EOL> def populate ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . exists : <EOL> raise CacheAlreadyExistsException ( '<STR_LIT>' % self . cache_uri ) <EOL> self . _populate_setup ( ) <EOL> self . graph . open ( self . cache_uri , create = True ) <EOL> with closing ( self . graph ) : <EOL> with self . _download_metadata_archive ( ) as metadata_archive : <EOL> for fact in self . _iter_metadata_triples ( metadata_archive ) : <EOL> self . graph . add ( fact ) <EOL> def _populate_setup ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def refresh ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . exists : <EOL> self . delete ( ) <EOL> self . populate ( ) <EOL> self . open ( ) <EOL> @ property <EOL> def _local_storage_path ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . cache_uri <EOL> @ staticmethod <EOL> def _add_namespaces ( graph ) : <EOL> """<STR_LIT>""" <EOL> graph . bind ( '<STR_LIT>' , PGTERMS ) <EOL> graph . bind ( '<STR_LIT>' , DCTERMS ) <EOL> @ contextmanager <EOL> def _download_metadata_archive ( self ) : <EOL> """<STR_LIT>""" <EOL> with tempfile . NamedTemporaryFile ( delete = False ) as metadata_archive : <EOL> shutil . copyfileobj ( urlopen ( self . catalog_source ) , metadata_archive ) <EOL> yield metadata_archive . name <EOL> remove ( metadata_archive . name ) <EOL> @ staticmethod <EOL> def _metadata_is_invalid ( fact ) : <EOL> """<STR_LIT>""" <EOL> return any ( isinstance ( token , URIRef ) and '<STR_LIT:U+0020>' in token for token in fact ) <EOL> @ classmethod <EOL> def _iter_metadata_triples ( cls , metadata_archive_path ) : <EOL> """<STR_LIT>""" <EOL> pg_rdf_regex = re . compile ( r'<STR_LIT>' ) <EOL> with closing ( tarfile . open ( metadata_archive_path ) ) as metadata_archive : <EOL> for item in metadata_archive : <EOL> if pg_rdf_regex . search ( item . name ) : <EOL> with disable_logging ( ) : <EOL> extracted = metadata_archive . extractfile ( item ) <EOL> graph = Graph ( ) . parse ( extracted ) <EOL> for fact in graph : <EOL> if cls . _metadata_is_invalid ( fact ) : <EOL> logging . info ( '<STR_LIT>' , fact ) <EOL> else : <EOL> yield fact <EOL> class SleepycatMetadataCache ( MetadataCache ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cache_location ) : <EOL> self . _check_can_be_instantiated ( ) <EOL> cache_uri = cache_location <EOL> store = '<STR_LIT>' <EOL> MetadataCache . __init__ ( self , store , cache_uri ) <EOL> def _populate_setup ( self ) : <EOL> makedirs ( self . cache_uri ) <EOL> @ classmethod <EOL> def _check_can_be_instantiated ( cls ) : <EOL> try : <EOL> from bsddb import db <EOL> except ImportError : <EOL> try : <EOL> from bsddb3 import db <EOL> except ImportError : <EOL> db = None <EOL> if db is None : <EOL> raise InvalidCacheException ( '<STR_LIT>' ) <EOL> del db <EOL> class SqliteMetadataCache ( MetadataCache ) : <EOL> """<STR_LIT>""" <EOL> _CACHE_URI_PREFIX = '<STR_LIT>' <EOL> def __init__ ( self , cache_location ) : <EOL> cache_uri = self . _CACHE_URI_PREFIX + cache_location <EOL> store = plugin . get ( '<STR_LIT>' , Store ) ( identifier = _DB_IDENTIFIER ) <EOL> MetadataCache . __init__ ( self , store , cache_uri ) <EOL> @ property <EOL> def _local_storage_path ( self ) : <EOL> return self . cache_uri [ len ( self . _CACHE_URI_PREFIX ) : ] <EOL> _METADATA_CACHE = None <EOL> def set_metadata_cache ( cache ) : <EOL> """<STR_LIT>""" <EOL> global _METADATA_CACHE <EOL> if _METADATA_CACHE and _METADATA_CACHE . is_open : <EOL> _METADATA_CACHE . close ( ) <EOL> _METADATA_CACHE = cache <EOL> def get_metadata_cache ( ) : <EOL> """<STR_LIT>""" <EOL> global _METADATA_CACHE <EOL> if _METADATA_CACHE is None : <EOL> _METADATA_CACHE = _create_metadata_cache ( _DB_PATH ) <EOL> return _METADATA_CACHE <EOL> def _create_metadata_cache ( cache_location ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return SleepycatMetadataCache ( cache_location ) <EOL> except InvalidCacheException : <EOL> logging . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> return SqliteMetadataCache ( cache_location ) <EOL> def load_metadata ( refresh_cache = False ) : <EOL> """<STR_LIT>""" <EOL> cache = get_metadata_cache ( ) <EOL> if refresh_cache : <EOL> cache . refresh ( ) <EOL> if not cache . is_open : <EOL> cache . open ( ) <EOL> return cache . graph </s>
