def dict_to_dcat(dataset_dict, portal, graph=None, format='json-ld', portal_uri=None):
    if graph == None:
        graph = rdflib.Graph()
    dataset_ref = None

    if portal.software == 'CKAN':
        converter = CKANConverter(graph, portal.apiuri)
        dataset_ref = converter.graph_from_ckan(dataset_dict)
    elif portal.software == 'Socrata':
        if 'dcat' in dataset_dict and dataset_dict['dcat']:
            graph.parse(data=dataset_dict['dcat'], format='xml')
            dataset_ref = fix_socrata_graph(graph, dataset_dict, portal.apiuri)
            # TODO redesign distribution, format, contact (publisher, organization)
    elif portal.software == 'OpenDataSoft':
        dataset_ref = graph_from_opendatasoft(graph, dataset_dict, portal.apiuri)
        # TODO contact, publisher, organization
    elif portal.software == 'XMLDCAT':
        # dataset_dict is already json-ld
        str_ds = json.dumps(dataset_dict)
        graph.parse(data=str_ds, format='json-ld')
        dataset_ref = graph.value(predicate=RDF.type, object=DCAT.Dataset)
    elif portal.software == 'DataGouvFr':
        dataset_ref = graph_from_data_gouv_fr(graph, dataset_dict, portal.apiuri)

    # add portal ref to graph
    if portal_uri and dataset_ref:
        graph.add((portal_uri, DCAT.dataset, dataset_ref))

    if portal.software == 'XMLDCAT':
        return dataset_dict
    else:
        return json.loads(graph.serialize(format=format)) DAQ = Namespace("http://purl.org/eis/vocab/daq#")
DCAT = Namespace("http://www.w3.org/ns/dcat#")
DQV = Namespace("http://www.w3.org/ns/dqv#")
DUV = Namespace("http://www.w3.org/ns/duv#")
OA = Namespace("http://www.w3.org/ns/oa#")
PROV = Namespace("http://www.w3.org/ns/prov#")
SDMX = Namespace("http://purl.org/linked-data/sdmx/2009/attribute#")
PWQ = Namespace("http://data.wu.ac.at/portalwatch/quality#")
PW_AGENT = URIRef("http://data.wu.ac.at/portalwatch")
def general_prov(graph):
def fetch_prov(snapshot, portalid, start_time, end_time, graph):
def quality_prov(entity, dataset, gen_time, fetch_activity, graph):
def add_dimensions_and_metrics(g):
def get_measures_and_dataset(portal, dataset, datasetquality, graph=None, portal_uri=None, fetch_activity=None):
def _get_measures_for_dataset(portal, dataset, datasetquality):
def get_measures_for_dataset(portal, dataset, datasetquality):
def dataset_quality_to_dqv(graph, ds_id, datasetquality, snapshot, fetch_activity=None): def parseDate(date):
    date_parts = []
    # YYYY<MM|DD|HH|MM|SS>
    if len(date) < 4:
        # error
        pass
    else:
        date_parts_length = [4, 2, 2, 2, 2, 2]
        date_parts = []
        for l in date_parts_length:
            if len(date) >= l:
                date_parts.append(int(date[0:l]))
                date = date[l:]

    if len(date_parts) == 0:
        return None
    elif len(date_parts) == 1:
        date_parts.append(1)
        date_parts.append(1)
    elif len(date_parts) == 2:
        date_parts.append(1)

    return datetime.datetime(*map(int, date_parts)) class Timer(object):

    measures={}

    def __init__(self, verbose=False, key=None):
        self.verbose = verbose
        self.key=key

    def __enter__(self):
        self.start = time.time()
        return self

    def __exit__(self, *args):
        end = time.time()
        secs = end - self.start
        msecs = secs * 1000  # millisecs
        if self.verbose:
            print '(%s) elapsed time: %f ms' % (self.key,msecs)
        if self.key:
            if self.key not in self.__class__.measures:
                self.__class__.measures[self.key]=faststat.Stats()
            if msecs>=0:
                self.__class__.measures[self.key].add(msecs)


    @classmethod
    def printStats(cls):
        print '>>>','--*'*10,'TIMER','*--'*10
        s=[" Timing stats:","\n"]
        for m,st in Timer.measures.items():
            p = st.percentiles
            if st.n < len(p):
                quartiles = "(n too small)"
            else:
                quartiles = (_sigfigs(p.get(0.25, -1)),
                    _sigfigs(p.get(0.5, -1)), _sigfigs(p.get(0.75, -1)))
            d = ["  ["+m+'] -', str(st.mean), 'avg ms for',m,str(st.n),'calls)'
                    ,"\n       (min:",str(st.min),"-",str(st.max),":max, quantils:",quartiles,")\n"]
            s=s+d
        s=s+['-'*50]
        print " ".join(s)
        print '<<<','--*'*10,'TIMER','*--'*10

    @classmethod
    def getStats(cls):

        stats={}
        for m,st in cls.measures.items():
            p = st.percentiles
            stats[m]={
                'avg':st.mean
                , 'calls':st.n
                , 'min':st.min
                , 'max':st.max
                , 'q25':None, 'q5':None, 'q75':None
            }
            if st.n < len(p):
                quartiles = "(n too small)"
            else:
                stats[m]['q25']=_sigfigs(p.get(0.25, -1))
                stats[m]['q5']=_sigfigs(p.get(0.5, -1))
                stats[m]['q75']=_sigfigs(p.get(0.75, -1))
        return stats def getPreviousWeek(snapshot):
    d = tofirstdayinisoweek(snapshot)
    prev = getSnapshotfromTime(d, timedelta(days=7), before=True)
    if snapshot == prev:
        return prev-1 def default_error_handler(error):
def database_not_found_error_handler(e): def row2dict(r):
    if hasattr(r, '_fields'):
        d = {}
        for field in r._fields:
            rf = r.__getattribute__(field)
            if isinstance(rf, Base):
                d.update(_row2dict(rf))
            else:
                d[field] = rf
        return d
    if isinstance(r, Base):
        return _row2dict(r) class Portal(Base):
    __tablename__ = tab_portals

    id      = Column(String, primary_key=True, index=True,nullable=False)
    uri     = Column(String, nullable=False)
    apiuri  = Column(String)
    software = Column(String(12), nullable=False) # OpenDataSoft, CKAN, Socrata <13
    iso     = Column(String(2), nullable=False)
    active  = Column(Boolean, default=True,nullable=False)

    snapshots = relationship("PortalSnapshot", back_populates="portal")
    snapshotsquality = relationship("PortalSnapshotQuality", back_populates="portal")

    @hybrid_property
    def snapshot_count(self):
        print len(self.snapshots)
        return len(self.snapshots)
    @snapshot_count.expression

    def snapshot_count(cls):
        return select([func.count(PortalSnapshot.snapshot)])\
            .where(PortalSnapshot.portalid == cls.id).label("snapshot_count")

    @hybrid_property
    def first_snapshot(self):
        print [s for s in self.snapshots]
        return min([s.snapshot for s in self.snapshots])

    @first_snapshot.expression
    def first_snapshot(cls):
        return select([func.min(PortalSnapshot.snapshot)])\
            .where(PortalSnapshot.portalid == cls.id).label("first_snapshot")

    @hybrid_property
    def last_snapshot(self):
        return max([s.snapshot for s in self.snapshots])

    @last_snapshot.expression
    def last_snapshot(cls):
        return select([func.max(PortalSnapshot.snapshot)])\
            .where(PortalSnapshot.portalid == cls.id).label("last_snapshot")

    @hybrid_property
    def datasetcount(self):
        return self.snapshots.order_by(PortalSnapshot.snapshot.desc()).one().datasetcount

    @datasetcount.expression
    def datasetcount(cls):
        q=select([PortalSnapshot.datasetcount])\
            .where(PortalSnapshot.portalid == cls.id).order_by(PortalSnapshot.snapshot.desc()).limit(1).label("datasetcount")
        return q

    @hybrid_property
    def resourcecount(self):
        return self.snapshots.order_by(PortalSnapshot.snapshot.desc()).one().resourcecount

    @resourcecount.expression
    def resourcecount(cls):
        q=select([PortalSnapshot.resourcecount])\
            .where(PortalSnapshot.portalid == cls.id).order_by(PortalSnapshot.snapshot.desc()).limit(1).label("resourcecount")
        return q

    def __repr__(self):
        return "<Portal(id=%s, uri='%s', apiuri='%s', software='%s', iso=%s)>" % (
            self.id, self.uri, self.apiuri, self.software, self.iso) class PortalSnapshotQuality(Base):
    __tablename__ = tab_portalsnapshotquality

    portalid      = Column(String, ForeignKey(tab_portals+'.id'), primary_key=True, index=True,nullable=False)
    snapshot= Column( SmallInteger, primary_key=True)
    portal  = relationship("Portal", back_populates="snapshotsquality")

    cocu = Column(Float)
    cocuN = Column(Integer)
    coce = Column(Float)
    coceN = Column(Integer)
    coda = Column(Float)
    codaN = Column(Integer)
    cofo = Column(Float)
    cofoN = Column(Integer)
    coli = Column(Float)
    coliN = Column(Integer)
    coac = Column(Float)
    coacN = Column(Integer)
    exda = Column(Float)
    exdaN = Column(Integer)
    exri = Column(Float)
    exriN = Column(Integer)
    expr = Column(Float)
    exprN = Column(Integer)
    exac = Column(Float)
    exacN = Column(Integer)
    exdi = Column(Float)
    exdiN = Column(Integer)
    exte = Column(Float)
    exteN = Column(Integer)
    exsp = Column(Float)
    exspN = Column(Integer)
    exco = Column(Float)
    excoN = Column(Integer)
    opfo = Column(Float)
    opfoN = Column(Integer)
    opma = Column(Float)
    opmaN = Column(Integer)
    opli = Column(Float)
    opliN = Column(Integer)
    datasets=Column(Integer)

    def __repr__(self):
        return "<PortalSnapshotQuality(id=%s, snapshot=%s, agg=%s)>" % (
            self.portalid, self.snapshot,  any([self.exda,self.coac,self.coce,self.cocu])) class PortalSnapshot(Base):
    __tablename__ = tab_portalsnapshot

    portalid      = Column(String, ForeignKey(tab_portals+'.id'), primary_key=True, index=True,nullable=False)
    snapshot= Column( SmallInteger, primary_key=True)
    portal  = relationship("Portal", back_populates="snapshots")

    start       = Column(TIMESTAMP)
    end         = Column(TIMESTAMP)
    status      = Column(SmallInteger)
    exc         = Column(String)
    datasetcount    = Column(Integer)
    datasetsfetched    = Column(Integer)
    resourcecount   = Column(Integer)

    @hybrid_property
    def fetchtime(self):
        return self.end-self.start

    datasets = relationship("Dataset", back_populates="portalsnapshot")

    def __repr__(self):
        return "<PortalSnapshot(id=%s, snapshot=%s, start=%s, end=%s, status=%s,ds=%s,res=%s)>" % (
            self.portalid, self.snapshot, self.start, self.end, self.status,self.datasetcount,self.resourcecount) class ResourceInfo(Base):
    __tablename__ = tab_resourcesinfo

    uri= Column(String, primary_key=True)
    snapshot= Column(SmallInteger, primary_key=True)
    timestamp= Column(TIMESTAMP)
    status=Column(SmallInteger)
    exc=Column(String)
    header=Column(JSONB)
    mime=Column(String)
    size=Column(BigInteger) class MetaResource(Base):
    __tablename__ = tab_resources

    uri = Column(String, primary_key=True, index=True)
    md5 = Column(String,ForeignKey(DatasetData.md5), primary_key=True,index=True )
    valid = Column(Boolean)
    format = Column(String)
    media = Column(String)
    size = Column(BigInteger)
    created = Column(TIMESTAMP)
    modified = Column(TIMESTAMP)


    dataset  = relationship("DatasetData", back_populates="resources")


    def info(cls):
        return select([ResourceInfo]).where(cls.uri == ResourceInfo.uri).where(cls.snapshot== ResourceInfo.snapshot)

    def __repr__(self):
        return "<Resource(uri=%s, dataset=%s)>" % (
            self.uri, self.md5) class Dataset(Base):
    __tablename__ = tab_datasets

    id           = Column( String, primary_key=True)
    snapshot     = Column( SmallInteger, primary_key=True, index=True)
    portalid     = Column( String, primary_key=True, index=True)
    organisation = Column(String, index=True)
    title        = Column(String, index=True)
    md5          = Column(String, ForeignKey(tab_datasetsdata+'.md5'), index=True)


    __table_args__ = (ForeignKeyConstraint([portalid, snapshot],
                                           [tab_portalsnapshot+'.portalid',tab_portalsnapshot+'.snapshot']),
                      {})

    portalsnapshot  = relationship("PortalSnapshot", back_populates="datasets")
    data  = relationship("DatasetData", back_populates="dataset")

    def __repr__(self):
        return "<Dataset(id=%s, portalid='%s', snapshot=%s, md5=%s)>" % (
            self.id, self.portalid, self.snapshot, self.md5) class DatasetData(Base):
    __tablename__ = tab_datasetsdata

    md5 = Column(String, primary_key=True, index=True, nullable=False)
    raw = Column(JSONB)
    dataset  = relationship("Dataset", back_populates="data")
    resources  = relationship("MetaResource", back_populates="dataset")

    modified = Column(TIMESTAMP)
    created = Column(TIMESTAMP)
    organisation = Column(String, index=True)
    license = Column(String, index=True)

    def __repr__(self):
        return "<DatasetData(md5=%s, data=%s)>" % (
            self.md5, self.raw is not None) class DatasetQuality(Base):
    __tablename__ = tab_datasetsquality

    md5     = Column(String, ForeignKey(DatasetData.md5), primary_key=True, index=True)
    cocu = Column(Boolean)
    coce = Column(Boolean)
    coda = Column(Float)
    cofo = Column(Float)
    coli = Column(Boolean)
    coac = Column(Boolean)

    exda = Column(Float)
    exri = Column(Float)
    expr = Column(Float)
    exac = Column(Boolean)
    exdi = Column(Float)
    exte = Column(Float)
    exsp = Column(Float)
    exco = Column(Boolean)

    opfo = Column(Float)
    opma = Column(Float)
    opli = Column(Boolean)


    data = relationship("DatasetData", backref=backref("quality", uselist=False))

    def __repr__(self):
        return "<DatasetQuality(md5=%s, assessment=%s)>" % (
            self.md5, any([self.exda,self.coac,self.coce,self.cocu])) DCAT = Namespace("http://www.w3.org/ns/dcat#")
FOAF = Namespace("http://xmlns.com/foaf/0.1/")
VCARD = Namespace("http://www.w3.org/2006/vcard/ns#")
def resp_party(g, doc, orga):
def convert(portal, data):
import json
import logging
from flask import Response
from odpw.core.dataset_converter import dict_to_dcat
from flask import make_response, request, current_app, jsonify
from flask_restplus import cors
from sqlalchemy import and_
from odpw.quality import dqv_export
from odpw.utils.datamonitor_utils import parseDate
from odpw.utils.timing import Timer
from odpw.utils.utils_snapshot import getPreviousWeek
from odpw.web_rest.rest.odpw_restapi import api
from odpw.core.db import row2dict
from odpw.core.model import Portal, PortalSnapshotQuality, PortalSnapshot, ResourceInfo, MetaResource, Dataset, \
    DatasetData, DatasetQuality
from flask import request
from flask_restplus import Resource
from schemadotorg import dcat_to_schemadotorg
# -*- coding: utf-8 -*-






log = logging.getLogger(__name__)

ns = api.namespace('portal', description='Operations related to a specific portal')

@ns.route('/<portalid>/<int:snapshot>/all')
@ns.doc(params={'portalid': 'A portal id', 'snapshot':'Snapshot in yyww format (e.g. 1639 -> 2016 week 39)'}, description="This API returns the portal specific information and the aggregated quality measures for a given portal and snapshot")
class PortalAll(Resource):
    def get(self, portalid,snapshot):
        q=PortalSnapshot.query
        if snapshot is not None:
            q=q.filter(PortalSnapshotQuality.snapshot==snapshot)
        q=q.filter(PortalSnapshot.portalid==portalid)\
            .outerjoin(PortalSnapshotQuality, and_(PortalSnapshot.portalid==PortalSnapshotQuality.portalid,PortalSnapshot.snapshot==PortalSnapshotQuality.snapshot))\
            .join(Portal)\
            .add_entity(PortalSnapshotQuality)\
            .add_entity(Portal)

        return jsonify([row2dict(i) for i in q.all()])


@ns.route('/<portalid>/<int:snapshot>/quality')
@ns.doc(params={'portalid': 'A portal id', 'snapshot':'Snapshot in yyww format (e.g. 1639 -> 2016 week 39)'}, description="This API returns the aggregated quality measures for a given portal and snapshot")
class PortalSnapshotQuality1(Resource):

    def get(self, portalid,snapshot):
        #print portalid, snapshot
        #with Timer(key="portalQuality",verbose=True):
        session=current_app.config['dbsession']

        q=session.query(PortalSnapshotQuality).filter(PortalSnapshotQuality.portalid==portalid).filter(PortalSnapshotQuality.snapshot==snapshot)
        data=[row2dict(r) for r in q.all()]

        return jsonify(data)


@ns.route('/<portalid>/<int:snapshot>/datasets')
@ns.doc(params={'portalid': 'A portal id', 'snapshot':'Snapshot in yyww format (e.g. 1639 -> 2016 week 39)'}, description="This API returns the full list of all datasets for a given portal and snapshot")
class PortalDatasets(Resource):

    #@cors.crossdomain(origin='*')
    def get(self, portalid,snapshot):
        with Timer(key="PortalDatasets.get",verbose=True):
            session=current_app.config['dbsession']

            q=session.query(Dataset)\
                .filter(Dataset.snapshot==snapshot)\
                .filter(Dataset.portalid==portalid)
            data=[row2dict(r) for r in q.all()]

            return jsonify(data)


@ns.route('/<portalid>/<int:snapshot>/dataset/<datasetid>')
@ns.doc(params={'portalid': 'A portal id', 'snapshot':'Snapshot in yyww format (e.g. 1639 -> 2016 week 30)','datasetid':'ID of dataset'})
class PortalDatasetData(Resource):

    #@cors.crossdomain(origin='*')
    def get(self, portalid,snapshot, datasetid):
        with Timer(key="PortalDatasetData.get",verbose=True):
            session=current_app.config['dbsession']

            q=session.query(DatasetData) \
                .join(Dataset, DatasetData.md5 == Dataset.md5) \
                .filter(Dataset.snapshot==snapshot)\
                .filter(Dataset.portalid==portalid)\
                .filter(Dataset.id == datasetid)
            data = [row2dict(r) for r in q.all()]

            return jsonify(data)


@ns.route('/<portalid>/<int:snapshot>/dataset/<datasetid>/dcat')
@ns.doc(params={'portalid': 'A portal id', 'snapshot':'Snapshot in yyww format (e.g. 1639 -> 2016 week 39)','datasetid':'ID of dataset'})
class PortalDatasetData(Resource):

    #@cors.crossdomain(origin='*')
    def get(self, portalid,snapshot, datasetid):
        with Timer(key="PortalDatasetData.get",verbose=True):
            session=current_app.config['dbsession']

            q=session.query(DatasetData) \
                .join(Dataset, DatasetData.md5 == Dataset.md5) \
                .filter(Dataset.snapshot==snapshot)\
                .filter(Dataset.portalid==portalid)\
                .filter(Dataset.id == datasetid)
            data=q.first()

            P= session.query(Portal).filter(Portal.id==portalid).first()
            if data == None:
                return make_response(jsonify({'error': 'Dataset not found'}), 500)

            return jsonify(dict_to_dcat(data.raw, P))