def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):  # pragma: no cover
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, str):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, str):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers

            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)

    return decorator def extract_first_geometry_fragment(geojson):
    """Given a geojson document, return a geojson geometry fragment marked as 4326
    encoding. If there are multiple features in the document, just make a
    fragment of the first feature. This is what PostGIS's ST_GeomFromGeoJSON
    expects.

    :param geojson: A full geojson document
    :type geojson: str
    :return: dict representing geojson structure
    """
    geo = json.loads(geojson)
    if 'features' in list(geo.keys()):
        fragment = geo['features'][0]['geometry']
    elif 'geometry' in list(geo.keys()):
        fragment = geo['geometry']
    else:
        fragment = geo

    return fragment def make_fragment_str(geojson_fragment, buffer=100):
    if geojson_fragment['type'] == 'LineString':
        shape = asShape(geojson_fragment)
        lat = shape.centroid.y
        x, y = get_size_in_degrees(buffer, lat)
        geojson_fragment = shape.buffer(y).__geo_interface__

    geojson_fragment['crs'] = {
        'type': 'name',
        'properties': {'name': 'EPSG:4326'}
    }

    return json.dumps(geojson_fragment) def parse_tree(table, condition_tree, literally=False):
    """Parse nested conditions provided as a dict for a single table. Wraps
    _parse_condition_tree and raises a ValueError if it fails.

    :param table: table object whose columns are being used in the conditions
    :param condition_tree: dictionary of conditions created from JSON
    :param literally: whether or not to create conditions as literal strings
    :returns SQLAlchemy conditions for querying the table with
    """
    try:
        return _parse_condition_tree(table, condition_tree, literally)
    except Exception as ex:
        raise ValueError('{} caused parse to fail for table {} with args {}'
                         .format(ex, table, condition_tree)) def make_job_response(endpoint, validated_query):
    msg = 'This feature, enabled by the jobs=true flag, is currently ' \
          'undergoing maintenance, we apologize for any inconvenience.'
    return jsonify({'unavailable': msg}) def detail_query(args, aggregate=False):
    meta_params = ('dataset', 'shapeset', 'data_type', 'geom', 'obs_date__ge',
                   'obs_date__le')
    meta_vals = (args.data.get(k) for k in meta_params)
    dataset, shapeset, data_type, geom, obs_date__ge, obs_date__le = meta_vals

    # If there aren't tree filters provided, a little formatting is needed
    # to make the general filters into an 'and' tree.
    if not has_tree_filters(args.data):
        # Creates an AND condition tree and adds it to args.
        args.data[dataset.name + '__filter'] = request_args_to_condition_tree(
            request_args=args.data,
            ignore=['shapeset']
        )

    # Sort out the filter conditions from the rest of the user arguments.
    filters = {k: v for k, v in list(args.data.items()) if 'filter' in k}

    # Get upset if they specify more than a dataset and shapeset filter.
    if len(filters) > 2:
        return api_response.bad_request('Too many table filters provided.')

    # Query the point dataset.
    q = postgres_session.query(dataset)

    # If the user specified a geom, filter results to those within its shape.
    if geom:
        q = q.filter(dataset.c.geom.ST_Within(
            sqlalchemy.func.ST_GeomFromGeoJSON(geom)
        ))

    # Retrieve the filters and build conditions from them if they exist.
    point_ctree = filters.get(dataset.name + '__filter')

    # If the user specified point dataset filters, parse and apply them.
    if point_ctree:
        point_conditions = parse_tree(dataset, point_ctree)
        q = q.filter(point_conditions)

        # To allow both obs_date meta params and filter trees.
        q = q.filter(dataset.c.point_date >= obs_date__ge) if obs_date__ge else q
        q = q.filter(dataset.c.point_date <= obs_date__le) if obs_date__le else q

    # If a user specified a shape dataset, it was either through the /shapes
    # enpoint, which uses the aggregate result, or through the /detail endpoint
    # which uses the joined result.
    if shapeset is not None:
        if aggregate:
            q = q.from_self(shapeset).filter(dataset.c.geom.ST_Intersects(shapeset.c.geom)).group_by(shapeset)
        else:
            shape_columns = ['{}.{} as {}'.format(shapeset.name, col.name, col.name) for col in shapeset.c]
            q = q.join(shapeset, dataset.c.geom.ST_Within(shapeset.c.geom))
            q = q.add_columns(*shape_columns)

        # If there's a filter specified for the shape dataset, apply those conditions.
        shape_ctree = filters.get(shapeset.name + '__filter')
        if shape_ctree:
            shape_conditions = parse_tree(shapeset, shape_ctree)
            q = q.filter(shape_conditions)

    return q def aggregate_point_data_response(data_type, rows, dataset_names):
    if data_type == 'csv':
        return form_csv_detail_response(['hash', 'ogc_fid'], rows, dataset_names)
    else:
        return form_geojson_detail_response(['hash', 'ogc_fid'], rows) def bad_request(msg):
    return make_error(msg, 400) def export_dataset_to_response(shapeset, data_type, query=None):
    export_format = str.lower(str(data_type))

    # Make a filename that we are reasonably sure to be unique and not occupied by anyone else.
    sacrifice_file = tempfile.NamedTemporaryFile()
    export_path = sacrifice_file.name
    sacrifice_file.close()  # Removes file from system.

    try:
        # Write to that filename.
        OgrExport(export_format, export_path, shapeset.name, query).write_file()
        # Dump it in the response.
        with open(export_path, 'rb') as to_export:
            resp = make_response(to_export.read(), 200)

        extension = _shape_format_to_file_extension(export_format)

        # Make the downloaded filename look nice
        shapemeta = ShapeMetadata.get_by_dataset_name(shapeset.name)
        resp.headers['Content-Type'] = _shape_format_to_content_header(export_format)
        resp.headers['Content-Disposition'] = "attachment; filename='{}.{}'".format(shapemeta.human_name, extension)
        return resp

    except Exception as e:
        error_message = 'Failed to export shape dataset {}'.format(shapeset.name)
        print((repr(e)))
        return make_response(error_message, 500)
    finally:
        # Don't leave that file hanging around.
        if os.path.isfile(export_path):
            os.remove(export_path) def make_error(msg, status_code, arguments=None):

    if not arguments:
        arguments = request.args

    resp = {
        'meta': {
            'status': 'error',
            'message': msg,
            'query': arguments
        },
        'objects': [],
    }

    response = jsonify(resp)
    response.status_code = status_code
    return response class ExportFormatsValidator(Validator):
    """For /shapes/<shapeset_name>?data_type=<format>
    """
    valid_formats = {'shapefile', 'kml', 'json'}
    data_type = fields.Str(default='json', validate=OneOf(valid_formats)) class Validator(Schema):
    """Base validator object using Marshmallow. Don't be intimidated! As scary
    as the following block of code looks it's quite simple, and saves us from
    writing validators. Let's break it down...

    <FIELD_NAME> = fields.<TYPE>(default=<DEFAULT_VALUE>, validate=<VALIDATOR FN>)

    The validator, when instantiated, has a method called 'dump'.which expects a
    dictionary of arguments, where keys correspond to <FIELD_NAME>. The validator
    has a default <TYPE> checker, that along with extra <VALIDATOR FN>s will
    accept or reject the value associated with the key. If the value is missing
    or rejected, the validator will substitute it with the value specified by
    <DEFAULT_VALUE>.
    """
    valid_aggs = {'day', 'week', 'month', 'quarter', 'year'}
    valid_formats = {'csv', 'geojson', 'json'}

    agg = fields.Str(default='week', validate=OneOf(valid_aggs))
    buffer = fields.Integer(default=100, validate=Range(0))
    dataset_name = fields.Str(default=None, validate=validate_dataset, dump_to='dataset')
    shape = fields.Str(default=None, validate=validate_shapeset, dump_to='shapeset')
    dataset_name__in = fields.List(fields.Str(), validate=validate_many_datasets)
    date__time_of_day_ge = fields.Integer(default=0, validate=Range(0, 23))
    date__time_of_day_le = fields.Integer(default=23, validate=Range(0, 23))
    data_type = fields.Str(default='json', validate=OneOf(valid_formats))
    location_geom__within = fields.Str(default=None, dump_to='geom', validate=validate_geom)
    obs_date__ge = fields.DateTime(default=datetime.now() - timedelta(days=90))
    obs_date__le = fields.DateTime(default=datetime.now())
    limit = fields.Integer(default=1000, validate=Range(0, 10000))
    offset = fields.Integer(default=0, validate=Range(0))
    resolution = fields.Integer(default=500, validate=Range(0))
    job = fields.Bool(default=False)
    all = fields.Bool(default=False) def has_tree_filters(request_args):
    """See if there are any <DATASET>__filter parameters.

    :param request_args: dictionary of request arguments
    :returns: boolean, true if there's a filter argument
    """
    return any('filter' in key for key in list(request_args.keys())) def validate(validator, request_args):
    """Validate a dictionary of arguments. Substitute all missing fields with
    defaults if not explicitly told to do otherwise.

    :param validator: what kind of validator to use
    :param request_args: dictionary of arguments from a request object
    :returns: ValidatorResult namedtuple
    """
    args = request_args.copy()

    result = marshmallow_validate(validator, args)

    # Holds messages concerning unnecessary parameters. These can be either
    # junk parameters, or redundant column parameters if a tree filter was
    # used.
    warnings = []

    # At this point validation splits. We can either validate tree-style column
    # arguments or validate them individually. We don't do both.

    # Determine unchecked parameters provided in the request.
    unchecked = set(args.keys()) - set(validator.fields.keys())

    # If tree filters were provided, ignore ALL unchecked parameters that are
    # not tree filters or response format information.
    if has_tree_filters(request_args):

        for key in request_args:
            value = args[key]
            if 'filter' in key:
                # This pattern matches the last occurrence of the '__' pattern.
                # Prevents an error that is caused by dataset names with trailing
                # underscores.
                t_name = re.split(r'__(?!_)', key)[0]

                # Report a filter which specifies a non-existent tree.
                try:
                    table = MetaTable.get_by_dataset_name(t_name).point_table
                except (AttributeError, NoSuchTableError):
                    try:
                        table = ShapeMetadata.get_by_dataset_name(t_name).shape_table
                    except (AttributeError, NoSuchTableError):
                        result.errors[t_name] = 'Table name {} could not be found.'.format(t_name)
                        return result

                # Report a tree which causes the JSON parser to fail.
                # Or a tree whose value is not valid.
                try:
                    cond_tree = json.loads(value)
                    if valid_tree(table, cond_tree):
                        result.data[key] = cond_tree
                except (ValueError, KeyError) as err:
                    result.errors[t_name] = 'Bad tree: {} -- causes error {}.'.format(value, err)
                    return result

            # These keys just have to do with the formatting of the JSON response.
            # We keep these values around even if they have no effect on a condition
            # tree.
            elif key in {'geom', 'offset', 'limit', 'agg', 'obs_date__le', 'obs_date__ge'}:
                pass

            # These keys are also ones that should be passed over when searching for
            # unused params. They are used, just in different forms later on, so no need
            # to report them.
            elif key in {'shape', 'dataset_name', 'dataset_name__in'}:
                pass

            # If the key is not a filter, and not used to format JSON, report
            # that we ignored it.
            else:
                warnings.append('Unused parameter {}, you cannot specify both '
                                'column and filter arguments.'.format(key))

    # If no tree filters were provided, see if any of the unchecked parameters
    # are usable as column conditions.
    else:
        try:
            table = result.data['dataset']
        except KeyError:
            table = result.data.get('shapeset')
        for param in unchecked:
            field = param.split('__')[0]
            if table is not None:
                try:
                    value = args[param]

                    valid_column_condition(table, field, value)
                    result.data[param] = args[param]
                except KeyError:
                    warnings.append('Unused parameter value {}={!r}'.format(param, value))
                    warnings.append('{} is not a valid column for {}'.format(param, table))
                except ValueError:
                    warnings.append('Unused parameter value {}={!r}'.format(param, value))
                    warnings.append('{} is not a valid value for {}'.format(args[param], param))

    # ValidatorResult(dict, dict, list)
    return ValidatorResult(result.data, result.errors, warnings) class ShapeMetadata(postgres_base):
    __tablename__ = 'meta_shape'

    dataset_name = Column(String, primary_key=True)
    human_name = Column(String, nullable=False)
    source_url = Column(String)
    view_url = Column(String)
    date_added = Column(Date, nullable=False)

    # Organization that published this dataset
    attribution = Column(String)
    description = Column(Text)
    update_freq = Column(String(100), nullable=False)

    # Who submitted this dataset?
    contributor_name = Column(String)
    contributor_organization = Column(String)
    contributor_email = Column(String)

    # Has an admin signed off on it?
    approved_status = Column(Boolean)

    # We always ingest geometric data as 4326
    bbox = Column(Geometry('POLYGON', srid=4326))
    # How many shape records are present?
    num_shapes = Column(Integer)
    # False when admin first submits metadata.
    # Will become true if ETL completes successfully.
    is_ingested = Column(Boolean, nullable=False)
    # foreign key of celery task responsible for shapefile's ingestion
    celery_task_id = Column(String)

    @classmethod
    def get_by_dataset_name(cls, name):
        shape_metatable = postgres_session.query(cls).filter(cls.dataset_name == name).first()
        return shape_metatable

    @classmethod
    def get_all_with_etl_status(cls):
        """
        :return: Every row of meta_shape joined with celery task status.
        """
        shape_query = """
            SELECT meta.*, c.*
            FROM meta_shape as meta
            LEFT JOIN celery_taskmeta as c
            ON c.task_id = meta.celery_task_id
            WHERE meta.approved_status = TRUE;
        """

        return list(postgres_session.execute(shape_query))

    @classmethod
    def index(cls, geom=None):
        # The attributes that we want to pass along as-is
        as_is_attr_names = ['dataset_name', 'human_name', 'date_added',
                            'attribution', 'description', 'update_freq',
                            'view_url', 'source_url', 'num_shapes']

        as_is_attrs = [getattr(cls, name) for name in as_is_attr_names]

        # We need to apply some processing to the bounding box
        bbox = func.ST_AsGeoJSON(cls.bbox)
        attr_names = as_is_attr_names + ['bbox']
        attrs = as_is_attrs + [bbox]

        result = postgres_session.query(*attrs).filter(cls.is_ingested)
        listing = [dict(list(zip(attr_names, row))) for row in result]

        for dataset in listing:
            dataset['date_added'] = str(dataset['date_added'])

        if geom:
            listing = cls.add_intersections_to_index(listing, geom)

        listing = cls._add_fields_to_index(listing)

        return listing

    @classmethod
    def simple_index(cls, geom):
        column_names = {
            'dataset_name',
            'human_name',
            'date_added',
            'attribution',
            'description',
            'update_freq',
            'view_url',
            'source_url',
            'num_shapes'
        }

        columns = [getattr(cls, n) for n in column_names]
        columns.append(func.ST_AsGeoJson(cls.bbox).label('bbox'))

        query = postgres_session.query(*columns).filter(
            cls.bbox.ST_Intersects(
                func.ST_GeomFromGeoJSON(geom)
            )
        )

        results = [dict(zip(row.keys(), row)) for row in query]

        return cls._add_fields_to_index(results)

    @classmethod
    def _add_fields_to_index(cls, listing):
        for dataset in listing:
            name = dataset['dataset_name']
            try:
                # Reflect up the shape table
                table = Table(name,
                              postgres_base.metadata,
                              autoload=True,
                              extend_existing=True)
            except NoSuchTableError:
                # If that table doesn't exist (?!?!)
                # don't try to form the fields.
                continue

            finally:
                # Extract every column's info.
                fields_list = []
                for col in table.columns:
                    if not isinstance(col.type, NullType):
                        # Don't report our internal-use columns
                        if col.name in {'geom', 'ogc_fid', 'hash'}:
                            continue
                        field_object = {
                            'field_name': col.name,
                            'field_type': str(col.type)
                        }
                        fields_list.append(field_object)
                dataset['columns'] = fields_list
        return listing

    @classmethod
    def tablenames(cls):
        return [x.dataset_name for x in postgres_session.query(ShapeMetadata.dataset_name).all()]

    @staticmethod
    def add_intersections_to_index(listing, geom):
        # For each dataset_name in the listing,
        # get a count of intersections
        # and replace num_geoms

        for row in listing:
            name = row['dataset_name']
            num_intersections_query = """
            SELECT count(g.geom) as num_geoms
            FROM "{dataset_name}" as g
            WHERE ST_Intersects(g.geom, ST_GeomFromGeoJSON('{geojson_fragment}'))
            """.format(dataset_name=name, geojson_fragment=geom)

            num_intersections = postgres_session.execute(num_intersections_query) \
                .first().num_geoms
            row['num_shapes'] = num_intersections

        intersecting_rows = [row for row in listing if row['num_shapes'] > 0]
        return intersecting_rows

    @classmethod
    def get_metadata_with_etl_result(cls, table_name):
        query = """
            SELECT meta.*, celery.status, celery.traceback, celery.date_done
            FROM meta_shape as meta
            LEFT JOIN celery_taskmeta as celery
            ON celery.task_id = meta.celery_task_id
            WHERE meta.dataset_name='{}';
        """.format(table_name)

        metadata = postgres_session.execute(query).first()
        return metadata

    @classmethod
    def get_by_human_name(cls, human_name):
        return postgres_session.query(cls).get(cls.make_table_name(human_name))

    @classmethod
    def make_table_name(cls, human_name):
        return slugify(human_name)

    @classmethod
    def add(cls, human_name, source_url, approved_status, **kwargs):
        table_name = ShapeMetadata.make_table_name(human_name)
        new_shape_dataset = ShapeMetadata(
            # Required params
            dataset_name=table_name,
            human_name=human_name,
            source_url=source_url,
            approved_status=approved_status,
            # Params that reflect just-submitted, not yet ingested status.
            is_ingested=False,
            bbox=None,
            num_shapes=None,
            date_added=datetime.now().date(),
            # The rest
            **kwargs)
        postgres_session.add(new_shape_dataset)
        return new_shape_dataset

    @property
    def shape_table(self):
        try:
            return self._shape_table
        except AttributeError:
            self._shape_table = Table(self.dataset_name, postgres_base.metadata, autoload=True, extend_existing=True)
            return self._shape_table

    def remove_table(self):
        if self.is_ingested:
            drop = 'DROP TABLE {};'.format(self.dataset_name)
            postgres_session.execute(drop)
        postgres_session.delete(self)

    def update_after_ingest(self):
        self.is_ingested = True
        self.bbox = self._make_bbox()
        self.num_shapes = self._get_num_shapes()

    def _make_bbox(self):
        bbox_query = 'SELECT ST_Envelope(ST_Union(geom)) FROM {};'. \
            format(self.dataset_name)
        box = postgres_session.execute(bbox_query).first().st_envelope
        return box

    def _get_num_shapes(self):
        table = self.shape_table
        # Arbitrarily select the first column of the table to count against
        count_query = select([func.count(table.c.geom)])
        # Should return only one row.
        # And we want the 0th and only attribute of that row (the count).
        return postgres_session.execute(count_query).fetchone()[0]
import json
from collections import OrderedDict
from flask import make_response, request
from sqlalchemy import func
from sqlalchemy.exc import NoSuchTableError
from plenario.api.common import crossdomain, extract_first_geometry_fragment, make_fragment_str
from plenario.api.condition_builder import parse_tree
from plenario.api.jobs import make_job_response
from plenario.api.point import detail_query
from plenario.api.response import aggregate_point_data_response, bad_request, export_dataset_to_response, make_error
from plenario.api.validator import ExportFormatsValidator, Validator, has_tree_filters, validate
from plenario.models import ShapeMetadata




@crossdomain(origin='*')
def get_all_shape_datasets():
    """Fetches metadata for every shape dataset in meta_shape.
    """
    try:
        response_skeleton = {
            'meta': {
                'status': 'ok',
                'message': '',
            },
            'objects': []
        }

        geom = request.args.get('location_geom__within')
        simple_bbox = request.args.get('simple_bbox')

        if geom:
            geom = make_fragment_str(
                extract_first_geometry_fragment(geom)
            )

        if simple_bbox:
            public_listing = ShapeMetadata.simple_index(geom)
        else:
            public_listing = ShapeMetadata.index(geom)
        response_skeleton['objects'] = public_listing
        status_code = 200

    except Exception as e:
        response_skeleton = {
            'meta': {
                'status': 'error',
                'message': str(e),
            }
        }
        status_code = 500

    resp = make_response(json.dumps(response_skeleton, default=str), status_code)
    resp.headers['Content-Type'] = 'application/json'
    return resp


@crossdomain(origin='*')
def aggregate_point_data(point_dataset_name, polygon_dataset_name):
    consider = ('dataset_name', 'shape', 'obs_date__ge', 'obs_date__le',
                'data_type', 'location_geom__within', 'job')

    request_args = request.args.to_dict()
    request_args['dataset_name'] = point_dataset_name
    request_args['shape'] = polygon_dataset_name

    validated_args = validate(Validator(only=consider), request_args)

    if validated_args.errors:
        return bad_request(validated_args.errors)
    elif validated_args.data.get('job'):
        return make_job_response('aggregate-point-data', validated_args)
    else:
        result = _aggregate_point_data(validated_args)
        data_type = validated_args.data.get('data_type')

        return aggregate_point_data_response(