RESPONSE_LIMIT = 1000
CACHE_TIMEOUT = 60 * 60 * 6
def unknown_object_json_handler(obj):
def date_json_handler(obj):
def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):  # pragma: no cover
    def get_methods():
    def decorator(f):
        def wrapped_function(*args, **kwargs):
def make_cache_key(*args, **kwargs):
def make_csv(data):
def extract_first_geometry_fragment(geojson):
def make_fragment_str(geojson_fragment, buffer=100): def parse_tree(table, condition_tree, literally=False):
    """Parse nested conditions provided as a dict for a single table. Wraps
    _parse_condition_tree and raises a ValueError if it fails.

    :param table: table object whose columns are being used in the conditions
    :param condition_tree: dictionary of conditions created from JSON
    :param literally: whether or not to create conditions as literal strings
    :returns SQLAlchemy conditions for querying the table with
    """
    try:
        return _parse_condition_tree(table, condition_tree, literally)
    except Exception as ex:
        raise ValueError('{} caused parse to fail for table {} with args {}'
                         .format(ex, table, condition_tree)) def valid_tree(table, tree):
    """Given a dictionary containing a condition tree, validate all conditions
    nestled in the tree.

    :param table: table to build conditions for, need it for the columns
    :param tree: condition_tree
    :returns: boolean value, true if the tree is valid
    """
    if not list(tree.keys()):
        raise ValueError('Empty or malformed tree.')

    op = tree.get('op')
    if not op:
        raise ValueError('Invalid keyword in {}'.format(tree))

    if op == 'and' or op == 'or':
        return all([valid_tree(table, subtree) for subtree in tree['val']])

    elif op in field_ops:
        col = tree.get('col')
        val = tree.get('val')

        if col is None or val is None:
            err_msg = 'Missing or invalid keyword in {}'.format(tree)
            err_msg += " -- use format '{\"op\": OP, \"col\": COL, \"val\": VAL}'"
            raise ValueError(err_msg)

        return valid_column_condition(table, col, val)

    else:
        raise ValueError('Invalid operation {}'.format(op)) def create_database(bind: Engine, database: str) -> None:
def drop_database(bind: Engine, database: str) -> None:
def create_extension(bind: Engine, extension: str) -> None:
def psql(path: str) -> None:
def postgres_session_context():
def redshift_session_context(): class FeatureMeta(postgres_base):
    __tablename__ = 'sensor__feature_metadata'

    name = Column(String, primary_key=True)
    networks = relationship('NetworkMeta', secondary='sensor__feature_to_network')
    observed_properties = Column(JSONB)

    def types(self):
        """Return a dictionary with the properties mapped to their types.
        """
        return {e['name']: e['type'] for e in self.observed_properties}

    def sensors(self) -> set:
        """Return the set of sensors that report on this feature.
        """
        results = set()
        for network in self.networks:
            for node in network.tree().values():
                for sensor, properties in node.items():
                    if self.name in {p.split('.')[0] for p in properties}:
                        results.add(sensor)

        return results

    @staticmethod
    def index(network_name=None):
        features = []
        for node in postgres_session.query(NodeMeta).all():
            if network_name is None or node.sensor_network.lower() == network_name.lower():
                for sensor in node.sensors:
                    for prop in sensor.observed_properties.values():
                        features.append(prop.split('.')[0].lower())
        return list(set(features))

    @staticmethod
    def properties_of(feature):
        query = postgres_session.query(FeatureMeta.observed_properties).filter(
            FeatureMeta.name == feature)
        return [feature + '.' + prop['name'] for prop in query.first().observed_properties]

    def mirror(self):
        """Create feature tables in redshift for all the networks associated
        with this feature.
        """
        for network in self.networks:
            self._mirror(network.name)

    def _mirror(self, network_name: str):
        """Create a feature table in redshift for the specified network.
        """
        columns = []
        for feature in self.observed_properties:
            column_name = feature['name']
            column_type = database_types[feature['type'].upper()]
            columns.append(Column(column_name, column_type, default=None))

        redshift_table = Table(
            '{}__{}'.format(network_name, self.name),
            redshift_base.metadata,
            Column('node_id', String, primary_key=True),
            Column('datetime', DateTime, primary_key=True),
            Column('meta_id', Float, nullable=False),
            Column('sensor', String, nullable=False),
            *columns,
            redshift_distkey='datetime',
            redshift_sortkey='datetime'
        )

        redshift_table.create()

    def __repr__(self):
        return '<Feature {!r}>'.format(self.name) class NetworkMeta(postgres_base):
    __tablename__ = 'sensor__network_metadata'

    name = Column(String, primary_key=True)
    nodes = relationship('NodeMeta')
    info = Column(JSONB)

    @staticmethod
    def index():
        networks = postgres_session.query(NetworkMeta)
        return [network.name.lower() for network in networks]

    def __repr__(self):
        return '<Network {!r}>'.format(self.name)

    def tree(self):
        sensor_tree_fn = sqla_fn.network_tree(self.name)
        sensor_tree_result_proxy = self.query.session.execute(sensor_tree_fn)
        return sensor_tree_result_proxy.scalar()

    def sensors(self) -> set:

        keys = []
        for sensor in self.tree().values():
            keys += sensor

        return set(keys)

    def features(self):

        keys = []
        for sensor in self.tree().values():
            for feature in sensor.values():
                keys += feature.values()

        return set([k.split('.')[0] for k in keys]) class NodeMeta(postgres_base):
    __tablename__ = 'sensor__node_metadata'

    id = Column(String, primary_key=True)
    sensor_network = Column(String, ForeignKey('sensor__network_metadata.name'), primary_key=True)
    location = Column(Geometry(geometry_type='POINT', srid=4326))
    sensors = relationship('SensorMeta', secondary='sensor__sensor_to_node')
    info = Column(JSONB)
    address = Column(String)

    column_editable_list = ('sensors', 'info')

    @staticmethod
    def all(network_name):
        query = NodeMeta.query.filter(NodeMeta.sensor_network == network_name)
        return query.all()

    @staticmethod
    def index(network_name):
        return [node.id for node in NodeMeta.all(network_name)]

    @staticmethod
    def nearest_neighbor_to(lng, lat, network, features):
        sensors = set()
        for feature in features:
            feature = FeatureMeta.query.get(feature)
            sensors = sensors | feature.sensors()

        return knn(
            lng=lng,
            lat=lat,
            network=network,
            sensors=sensors,
            k=10
        )

    @staticmethod
    def within_geojson(network: NetworkMeta, geojson: str):
        geom = sqla_fn.ST_GeomFromGeoJSON(geojson)
        within = NodeMeta.location.ST_Within(geom)
        query = NodeMeta.query.filter(within)
        query = query.filter(NodeMeta.sensor_network == network.name)
        return query

    @staticmethod
    def sensors_from_nodes(nodes):
        sensors_list = []
        for node in nodes:
            sensors_list += node.sensors
        return set(sensors_list)

    def features(self) -> set:
        feature_set = set()
        for feature in self.tree().values():
            feature_set.update(feature.keys())
        return feature_set

    def __repr__(self):
        return '<Node {!r}>'.format(self.id)

    def tree(self):
        return {s.name: s.tree() for s in self.sensors} class SensorMeta(postgres_base):
    __tablename__ = 'sensor__sensor_metadata'

    name = Column(String, primary_key=True)
    observed_properties = Column(JSONB)
    info = Column(JSONB)

    def features(self) -> set:
        """Return the features that this sensor reports on.
        """
        return {e.split('.')[0] for e in self.tree()}

    def __repr__(self):
        return '<Sensor {!r}>'.format(self.name)

    def tree(self):
        return {v: k for k, v in self.observed_properties.items()} def _fill_in_blanks(aggregates, agg_unit, start_dt, end_dt):
def _format_aggregates(aggregates, agg_label, agg_unit, start_dt, end_dt):
def _generate_aggregate_selects(table, target_columns, agg_fn, agg_unit):
def _generate_placeholder(aggreagte):
def _reflect(table_name, metadata, engine):
def _valid_columns(node, target_sensors, target_features, target_properties=None):
def _zero_out_datetime(dt, unit):
def aggregate(args, agg_label, agg_fn): def bad_request(msg):
    return make_error(msg, 400) def json_response_base(validator, data, query=''):
    meta = {
        'message': '',
        'total': len(data)
    }

    if validator:
        try:
            meta['message'] = validator.warnings
        except AttributeError:
            meta['message'] = None
        meta['query'] = query

    return {
        'meta': meta,
        'data': data
    } S3_BUCKET = get('S3_BUCKET', '') def reflect(table_name, metadata, engine):
    """Helper function for an oft repeated block of code.

    :param table_name: (str) table name
    :param metadata: (MetaData) SQLAlchemy object found in a declarative base
    :param engine: (Engine) SQLAlchemy object to send queries to the database
    :returns: (Table) SQLAlchemy object
    """
    return Table(
        table_name,
        metadata,
        autoload=True,
        autoload_with=engine
    )
import csv
import io
import json
import boto3
from datetime import datetime, timedelta
from flask import Response, jsonify, redirect, request, stream_with_context
from marshmallow import Schema
from marshmallow.exceptions import ValidationError
from marshmallow.fields import DateTime, Field, Float, Integer, List, String
from marshmallow.validate import Range
from shapely import wkb
from sqlalchemy import MetaData, and_, asc, desc, func as sqla_fn
from sqlalchemy.orm.exc import NoResultFound
from plenario.api.common import cache, crossdomain, extract_first_geometry_fragment, make_cache_key, make_fragment_str, \
    unknown_object_json_handler
from plenario.api.condition_builder import parse_tree
from plenario.api.validator import valid_tree
from plenario.database import redshift_base, redshift_engine, redshift_session
from plenario.models.SensorNetwork import FeatureMeta, NetworkMeta, NodeMeta, SensorMeta
from plenario.sensor_network.api.sensor_aggregate_functions import aggregate_fn_map
from plenario.sensor_network.api.sensor_response import bad_request, json_response_base
from plenario.settings import S3_BUCKET
from plenario.utils.helpers import reflect
        'nodes': nodes.split(',') if nodes else [],
        'sensors': sensors.split(',') if sensors else [],
    }

    validator = Validator()
    validated = validator.load(args)
    if validated.errors:
        return bad_request(validated.errors)

    return jsonify({'message': 'Your query params are good to go.'})


@crossdomain(origin='*')
def get_observations(network: str) -> Response:
    '''Return raw sensor network observations for a single feature within
    the specified network.

    :endpoint: /sensor-networks/<network-name>/query?feature=<feature>'''

    nodes = request.args.get('nodes')
    sensors = request.args.get('sensors')
    feature = request.args.get('feature')

    args = request.args.to_dict()
    args.update({
        'network': network,
        'feature': feature,
        'nodes': nodes.split(',') if nodes else [],
        'sensors': sensors.split(',') if sensors else [],
    })

    validator = Validator()
    validated = validator.load(args)
    if validated.errors:
        return bad_request(validated.errors)

    if '.' in feature:
        feature, property_ = feature.split('.', 1)
        validated.data.update({'property': property_})

    redshift_base.metadata.reflect()
    table = redshift_base.metadata.tables[network + '__' + feature]

    try:
        query = observation_query(table, **validated.data)
    except KeyError as err:
        return bad_request(str(err))

    data = list()
    for obs in query:
        data.append(format_observation(obs, table))

    return jsonify(json_response_base(validated, data, args))


@crossdomain(origin='*')
def get_observation_nearest(network: str) -> Response:
    '''Return a single observation from the node nearest to the specified
    long, lat coordinate.

    :endpoint: /sensor-networks/<network-name>/nearest
               ?lng=<lng>&lat=<lat>&feature=<feature>'''

    args = request.args.to_dict()
    args.update({'network': network})

    validator = NearestValidator(only=('lat', 'lng', 'feature', 'network', 'datetime', 'filter'))
    validated = validator.load(args)
    if validated.errors:
        return bad_request(validated.errors)

    result = get_observation_nearest_query(validated)
    return jsonify(json_response_base(validated, result, args))


@crossdomain(origin='*')
def get_observations_download(network: str) -> Response:
    '''Stream a sensor network's bulk records to a csv file.

    :endpoint: /sensor-networks/<network>/download'''

    nodes = request.args.get('node') or request.args.get('nodes')
    sensors = request.args.get('sensor') or request.args.get('sensors')
    features = request.args.get('feature') or request.args.get('features')

    kwargs = request.args.to_dict()
    kwargs.update({
        'network': network,
        'features': features.split(',') if features else [],
        'nodes': nodes.split(',') if nodes else [],
        'sensors': sensors.split(',') if sensors else [],
    })

    validator = NoLimitValidator()
    deserialized = validator.load(kwargs)
    if deserialized.errors:
        return bad_request(deserialized.errors)

    if not kwargs['features']:
        kwargs['features'] = deserialized.data['network'].features()

    deserialized = validator.load(kwargs)

    if not deserialized.data.get('start_datetime'):
        deserialized.data.update({'start_datetime': datetime.now() - timedelta(days=7)})

    if deserialized.data.get('data_type') == 'json':
        stream = get_observation_datadump_json(**deserialized.data)
        filename = datetime.now().isoformat() + '-' + deserialized.data['network'].name + '.json'
        attachment = Response(stream_with_context(stream), mimetype='text/json')
    else:
        stream = get_observation_datadump_csv(**deserialized.data)
        filename = datetime.now().isoformat() + '-' + deserialized.data['network'].name + '.csv'
        attachment = Response(stream_with_context(stream), mimetype='text/csv')

    content_disposition = 'attachment; filename={}'.format(filename)
    attachment.headers['Content-Disposition'] = content_disposition
    return attachment



@cache.cached(timeout=CACHE_TIMEOUT, key_prefix=make_cache_key)