pad-us / app.py
cassiebuhler's picture
fixed chart filters
3c693f3
# +
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# +
import streamlit as st
import streamlit.components.v1 as components
import base64
import leafmap.maplibregl as leafmap
import altair as alt
import ibis
from ibis import _
import ibis.selectors as s
from typing import Optional
def to_streamlit(
self,
width: Optional[int] = None,
height: Optional[int] = 600,
scrolling: Optional[bool] = False,
**kwargs,
):
try:
import streamlit.components.v1 as components
import base64
raw_html = self.to_html().encode("utf-8")
raw_html = base64.b64encode(raw_html).decode()
return components.iframe(
f"data:text/html;base64,{raw_html}",
width=width,
height=height,
scrolling=scrolling,
**kwargs,
)
except Exception as e:
raise Exception(e)
# gap codes 3 and 4 are off by default.
default_gap = {
3: False,
4: False,
}
# +
#pad_pmtiles = "https://data.source.coop/cboettig/pad-us-3/pad-stats.pmtiles"
#parquet = "https://data.source.coop/cboettig/pad-us-3/pad-stats.parquet"
pad_pmtiles = "https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/pad-stats.pmtiles"
# parquet = "https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/pad-stats.parquet"
parquet = "pad-stats.parquet"
# adding this to test out git
# some default color variables, consider user palette via st.color_picker()
private_color = "#DE881E" # orange #"#850101" # red
tribal_color = "#BF40BF" # purple
mixed_color = "#005a00" # green
public_color = "#3388ff" # blue
# default color breaks, consider tool via st.slider()
low = 2
high = 3
alpha = .5
color_choice = "Manager Type"
us_lower_48_area_m2 = 7.8e+12
# +
## Helper functions
#@st.cache_resource
# def ibis_connection(parquet):
# return ibis.read_parquet(parquet)
# pad_data = ibis_connection(parquet)
con = ibis.duckdb.connect(extensions=["spatial"])
pad_data = con.read_parquet(parquet)
#@st.cache_data()
# def summary_table(column, colors):
# df = (pad_data
# .rename(area = "area_square_meters")
# .group_by(_[column])
# .aggregate(
# )
# .mutate(percent_protected = _.percent_protected.round(1))
# .inner_join(colors, column)
# )
# df = df.to_pandas()
# df[column] = df[column].astype(str)
# return df
from functools import reduce
def get_summary(pad_data, combined_filter, column, colors=None): #summary stats, based on filtered data
# ca = ca.filter(_.gap_code.isin([1,2])) #only gap 1 and 2
df = pad_data.filter(combined_filter)
df = (df
.rename(area = "area_square_meters")
.group_by(*column) # unpack the list for grouping
.aggregate(hectares_protected = (_.area.sum() / 10000).round(),
percent_protected = 100 * _.area.sum() / us_lower_48_area_m2,
mean_richness = (_.richness * _.area).sum() / _.area.sum(),
mean_rsr = (_.rsr * _.area).sum() / _.area.sum(),
mean_irrecoverable_carbon = (_.irrecoverable_carbon * _.area).sum() / _.area.sum(),
mean_manageable_carbon = (_.manageable_carbon * _.area).sum() / _.area.sum(),
mean_carbon_lost = (_.deforest_carbon * _.area).sum() / _.area.sum(),
mean_crop_expansion = (_.crop_expansion * _.area).sum() / _.area.sum(),
mean_human_impact = (_.human_impact * _.area).sum() / _.area.sum(),
mean_forest_integrity_loss = (_.forest_integrity_loss*_.area).sum() / _.area.sum(),
mean_bio_intact_loss = (_.biodiversity_intactness_loss * _.area).sum() / _.area.sum(),
)
.mutate(percent_protected=_.percent_protected.round(1))
)
if colors is not None and not colors.empty: #only the df will have colors, df_tab doesn't since we are printing it.
df = df.inner_join(colors, column)
df = df.cast({col: "string" for col in column})
df = df.to_pandas()
return df
def summary_table(column, colors, filter_cols, filter_vals,colorby_vals): # get df for charts + df_tab for printed table + df_percent for percentage (only gap 1 and 2)
filters = []
if filter_cols and filter_vals: #if a filter is selected, add to list of filters
for filter_col, filter_val in zip(filter_cols, filter_vals):
if len(filter_val) > 1:
filters.append(getattr(_, filter_col).isin(filter_val))
else:
filters.append(getattr(_, filter_col) == filter_val[0])
if column not in filter_cols: #show color_by column in table by adding it as a filter (if it's not already a filter)
filter_cols.append(column)
filters.append(getattr(_, column).isin(colorby_vals[column]))
combined_filter = reduce(lambda x, y: x & y, filters) #combining all the filters into ibis filter expression
df = get_summary(pad_data, combined_filter, [column], colors) # df used for charts
df_tab = get_summary(pad_data, combined_filter, filter_cols, colors = None) #df used for printed table
df_percent = get_summary(pad_data.filter(_.gap_code.isin([1,2])), combined_filter, [column], colors) # only gap 1 and 2 count towards percentage
return df, df_tab, df_percent
def bar_chart(df, x, y):
chart = alt.Chart(df).mark_bar().encode(
x=x,
y=y,
color=alt.Color('color').scale(None)
).properties(width="container", height=300)
return chart
def area_plot(df, column):
base = alt.Chart(df).encode(
alt.Theta("percent_protected:Q").stack(True),
)
pie = ( base
.mark_arc(innerRadius= 40, outerRadius=100)
.encode(alt.Color("color:N").scale(None).legend(None),
tooltip=['percent_protected', 'hectares_protected', column])
)
text = ( base
.mark_text(radius=80, size=14, color="white")
.encode(text = column + ":N")
)
plot = pie # pie + text
return plot.properties(width="container", height=300)
def pad_style(paint, alpha):
return {
"version": 8,
"sources": {
"pad": {
"type": "vector",
"url": "pmtiles://" + pad_pmtiles,
"attribution": "US PAD v3"}},
"layers": [{
"id": "public",
"source": "pad",
"source-layer": "pad-stats",
"type": "fill",
"paint": {
"fill-color": paint,
"fill-opacity": alpha
}
}]}
def get_pmtiles_style(paint, alpha, cols, values): #style depends on the filters selected.
filters = []
for col, val in zip(cols, values):
filter_condition = ["match", ["get", col], val, True, False]
filters.append(filter_condition)
combined_filter = ["all"] + filters
return {
"version": 8,
"sources": {
"pad": {
"type": "vector",
"url": "pmtiles://" + pad_pmtiles,
"attribution": "US PAD v3",
}
},
"layers": [{
"id": "public",
"source": "pad",
"source-layer": "pad-stats",
"type": "fill",
"filter": combined_filter, # Use the combined filter
"paint": {
"fill-color": paint,
"fill-opacity": alpha
}
}]
}
# +
def getButtons(style_options, color_choice, default_gap=None): #finding the buttons selected to use as filters
column = style_options[color_choice]['property']
opts = [style[0] for style in style_options[color_choice]['stops']]
default_gap = default_gap or {}
buttons = {
name: st.checkbox(f"{name}", value=default_gap.get(name, True), key=column + str(name))
for name in opts
}
filter_choice = [key for key, value in buttons.items() if value] # return only selected
d = {}
d[column] = filter_choice
return d
def getColorVals(style_options, color_choice):
#df_tab only includes filters selected, we need to manually add "color_by" column (if it's not already a filter).
column = style_options[color_choice]['property']
opts = [style[0] for style in style_options[color_choice]['stops']]
d = {}
d[column] = opts
return d
custom_style = '''
"blue"
'''
sample_q = '''(
ibis.read_parquet(parquet).
mutate(area = _.area_square_meters).
group_by(_.gap_code).
aggregate(percent_protected = 100 * _.area.sum() / us_lower_48_area_m2,
mean_richness = (_.richness * _.area).sum() / _.area.sum(),
mean_rsr = (_.rsr * _.area).sum() / _.area.sum()
).
mutate(percent_protected = _.percent_protected.round())
)
'''
## Protected Area polygon color codes
manager = {
'property': 'manager_type',
'type': 'categorical',
'stops': [
['Federal', "darkblue"],
['State', public_color],
['Local Government', "lightblue"],
['Regional Agency Special District', "darkgreen"],
['Unknown', "grey"],
['Joint', "green"],
['American Indian Lands', tribal_color],
['Private', "darkred"],
['Non-Governmental Organization', "orange"]
]
}
easement = {
'property': 'category',
'type': 'categorical',
'stops': [
['Fee', public_color],
['Easement', private_color],
['Proclamation', tribal_color]
]
}
access = {
'property': 'public_access',
'type': 'categorical',
'stops': [
['Open Access', public_color],
['Closed', private_color],
['Unknown', "grey"],
['Restricted Access', tribal_color]
]
}
gap = {
'property': 'gap_code',
'type': 'categorical',
'stops': [
["1", "#26633d"],
["2", "#879647"],
["3", "#BBBBBB"],
["4", "#F8F8F8"]
]
}
iucn = {
'property': 'iucn_category',
'type': 'categorical',
'stops': [
["Ia: Strict nature reserves", "#4B0082"],
["Ib: Wilderness areas", "#663399"],
["II: National park", "#7B68EE"],
["III: Natural monument or feature", "#9370DB"],
["IV: Habitat / species management", "#8A2BE2"],
["V: Protected landscape / seascape", "#9932CC"],
["VI: Protected area with sustainable use of natural resources", "#9400D3"],
["Other Conservation Area", "#DDA0DD"],
["Unassigned", "#F8F8F8"],
]
}
style_options = {
"GAP Status Code": gap,
"IUCN Status Code": iucn,
"Manager Type": manager,
"Fee/Easement": easement,
"Public Access": access,
# "Mean Richness": richness,
# "Mean RSR": rsr,
# "custom": eval(custom)
}
code_ex='''
m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_expansion_100m_cog.tif",
palette="oranges", name="Cropland Expansion", transparent_bg=True, opacity = 0.7, fit_bounds=False)
'''
justice40 = "https://data.source.coop/cboettig/justice40/disadvantaged-communities.pmtiles"
justice40_fill = {
'property': 'Disadvan',
'type': 'categorical',
'stops': [
[0, "rgba(255, 255, 255, 0)"],
[1, "rgba(0, 0, 139, 1)"]]}
justice40_style = {
"version": 8,
"sources": {
"source1": {
"type": "vector",
"url": "pmtiles://" + justice40,
"attribution": "Justice40"}
},
"layers": [{
"id": "layer1",
"source": "source1",
"source-layer": "DisadvantagedCommunitiesCEJST",
"type": "fill",
"paint": {"fill-color": justice40_fill, "fill-opacity": 0.6}}]
}
bil_url = "https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/bil.geojson"
bil_fill = {
"fill-extrusion-color": {
"property": "AtlasCateg",
"type": "categorical",
"stops": [
["America the Beautiful Challenge Grants", "orange"],
["Clean Energy and Power", "gray"],
["Environmental Remediation", "green"],
["Resilience and Ecosystem Restoration", "purple"],
["Water Infrastructure", "blue"]
],
},
#"fill-extrusion-height": ["*", .01, ["get", "FundingAmo"]],
"fill-extrusion-height": ["*", 50, ["sqrt", ["get", "FundingAmo"]]],
"fill-extrusion-opacity": 0.9,
}
###########################################################################################################
# +
st.set_page_config(layout="wide", page_title="Protected Areas Explorer", page_icon=":globe:")
'''
# US Conservation Atlas Prototype
An interactive cloud-native geospatial tool for exploring and visualizing the United States' protected lands with open data.
- ❌ Safari/iOS not yet supported
- ⬅️ Use the left sidebar to color-code the map by different attributes, toggle on data layers and view summary charts, or filter data.
'''
st.divider()
filters = {}
m = leafmap.Map(style="positron")
# +
with st.sidebar:
with st.expander("🗺 Basemaps"):
# radio selector would make more sense
if st.toggle("Topography"):
m.add_basemap("Esri.WorldShadedRelief")
if st.toggle("Satellite"):
m.add_basemap("Esri.WorldImagery")
# if st.toggle("Protected Areas", True):
color_choice = st.radio("Color by:", style_options)
colorby_vals = getColorVals(style_options, color_choice) #get options for selected color_by column
alpha = st.slider("transparency", 0.0, 1.0, 0.5)
"Data layers:"
with st.expander("🦜 Biodiversity"):
a_bio = st.slider("transparency", 0.0, 1.0, 0.4, key = "biodiversity")
show_richness = st.toggle("Species Richness", False)
if show_richness:
m.add_tile_layer(url="https://data.source.coop/cboettig/mobi/tiles/red/species-richness-all/{z}/{x}/{y}.png",
name="MOBI Species Richness",
attribution="NatureServe",
opacity=a_bio
)
show_rsr = st.toggle("Range-Size Rarity")
if show_rsr:
m.add_tile_layer(url="https://data.source.coop/cboettig/mobi/tiles/green/range-size-rarity-all/{z}/{x}/{y}.png",
name="MOBI Range-Size Rarity",
attribution="NatureServe",
opacity=a_bio
)
#m.add_cog_layer("https://data.source.coop/cboettig/mobi/range-size-rarity-all/RSR_All.tif",
# palette="greens", name="Range-Size Rarity", transparent_bg=True, opacity = 0.9, fit_bounds=False)
with st.expander("⛅ Carbon & Climate"):
a_climate = st.slider("transparency", 0.0, 1.0, 0.3, key = "climate")
show_carbon_lost = st.toggle("Carbon Lost (2002-2022)")
if show_carbon_lost:
m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/deforest_carbon_100m_cog.tif",
palette="reds", name="Carbon Lost (2002-2022)", transparent_bg=True, opacity = a_climate, fit_bounds=False)
show_irr_carbon = st.toggle("Irrecoverable Carbon")
if show_irr_carbon:
m.add_cog_layer("https://data.source.coop/cboettig/carbon/cogs/irrecoverable_c_total_2018.tif",
palette="purples", name="Irrecoverable Carbon", transparent_bg=True, opacity = a_climate, fit_bounds=False)
show_man_carbon = st.toggle("Manageable Carbon")
if show_man_carbon:
m.add_cog_layer("https://data.source.coop/cboettig/carbon/cogs/manageable_c_total_2018.tif",
palette="greens", name="Manageable Carbon", transparent_bg=True, opacity = a_climate, fit_bounds=False)
with st.expander("🚜 Human Impacts"):
a_hi = st.slider("transparency", 0.0, 1.0, 0.5, key = "hi")
show_human_impact = st.toggle("Human Impact")
if show_human_impact:
hi="https://data.source.coop/vizzuality/hfp-100/hfp_2021_100m_v1-2_cog.tif"
m.add_cog_layer(hi, palette="purples", name="Human Impact", transparent_bg=True, opacity = a_hi, fit_bounds=False)
show_crop_expansion = st.toggle("Cropland Expansion")
if show_crop_expansion:
m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_expansion_100m_cog.tif",opacity = a_hi, name = "Cropland Expansion")
# palette="greens", name="cropland expansion", transparent_bg=True, opacity = 0.8, fit_bounds=False)
show_bio_loss = st.toggle("Biodiversity Intactness Loss")
if show_bio_loss:
m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_bii_100m_cog.tif",
palette="reds", name="Biodiversity Intactness Loss", transparent_bg=True, opacity = a_hi, fit_bounds=False)
show_forest_loss = st.toggle("Forest Integrity Loss")
if show_forest_loss:
m.add_cog_layer("https://data.source.coop/vizzuality/lg-land-carbon-data/natcrop_fii_100m_cog.tif",
palette="reds", name="Forest Integrity Loss", transparent_bg=True, opacity = a_hi, fit_bounds=False)
with st.expander("💰 Conservation Investment"):
if st.toggle("Bipartisan Infrastructure Law"):
m.add_geojson(bil_url, layer_type="fill-extrusion", paint=bil_fill, name="BIL", fit_bounds=False)
with st.expander("💻 Custom Code"):
if st.toggle("Custom Map Layers"):
code = st.text_area(label = "leafmap code:",
value = code_ex,
height = 100)
eval(compile(code, "<string>", "exec"))
st.divider()
"Filters:"
for label in style_options: # get selected filters (based on the buttons selected)
with st.expander(label):
if label == "GAP Status Code": # gap code 1 and 2 are on by default
opts = getButtons(style_options, label, default_gap)
else: # other buttons are not on by default.
opts = getButtons(style_options, label)
filters.update(opts)
selected = {k: v for k, v in filters.items() if v}
if selected:
filter_cols = list(selected.keys())
filter_vals = list(selected.values())
else:
filter_cols = []
filter_vals = []
style = get_pmtiles_style(style_options[color_choice], alpha, filter_cols, filter_vals)
legend_d = {cat: color for cat, color in style_options[color_choice]['stops']}
m.add_legend(legend_dict = legend_d, position = 'bottom-left')
m.add_pmtiles(pad_pmtiles, style=style, name="PAD", opacity=alpha, tooltip=True)
# style = get_pmtiles_style(style_options[color_choice], alpha)
# m.add_pmtiles(pad_pmtiles, style=style, visible=True, opacity=alpha, tooltip=True)
# "## Boundaries"
# boundaries = st.radio("Boundaries:",
# ["None",
# "State Boundaries",
# "County Boundaries",
# "Congressional District",
# "custom"]
# )
# +
# Map radio buttons to corresponding column:
select_column = {
"GAP Status Code": "gap_code",
"IUCN Status Code": "iucn_category",
"Manager Type": "manager_type",
"Fee/Easement": "category",
"Public Access": "public_access",
"Mean Richness": "gap_code",
"Mean RSR": "gap_code",
"custom": "gap_code"}
column = select_column[color_choice]
# Map radio buttons to corresponding color-scheme:
select_colors = {
"GAP Status Code": gap["stops"],
"IUCN Status Code": iucn["stops"],
"Manager Type": manager["stops"],
"Fee/Easement": easement["stops"],
"Public Access": access["stops"],
"Mean Richness": gap["stops"],
"Mean RSR": gap["stops"],
"custom": gap["stops"]}
colors = (ibis
.memtable(select_colors[color_choice], columns = [column, "color"])
.to_pandas()
)
# +
# get summary tables used for charts + printed table + percentage
# df - charts; df_tab - printed table (omits colors) + df_percent - only gap codes 1 & 2 count towards percentage
df,df_tab,df_percent = summary_table(column, colors, filter_cols, filter_vals, colorby_vals)
# compute area covered (only gap 1 and 2)
# df_onlygap = df[df.gap_code.isin([1,2])]
total_percent = df_percent.percent_protected.sum().round(1)
# charts displayed based on color_by variable
richness_chart = bar_chart(df, column, 'mean_richness')
rsr_chart = bar_chart(df, column, 'mean_rsr')
irr_carbon_chart = bar_chart(df, column, 'mean_irrecoverable_carbon')
man_carbon_chart = bar_chart(df, column, 'mean_manageable_carbon')
carbon_loss_chart = bar_chart(df, column, 'mean_carbon_lost')
hi_chart = bar_chart(df, column, 'mean_human_impact')
crop_expansion_chart = bar_chart(df, column, 'mean_crop_expansion')
bio_intact_loss_chart = bar_chart(df, column, 'mean_bio_intact_loss')
forest_integrity_loss_chart = bar_chart(df, column, 'mean_forest_integrity_loss')
main = st.container()
with main:
map_col, stats_col = st.columns([2,1])
with map_col:
to_streamlit(m, height=700)
# df = summary_table(column, colors)
# total_percent = df.percent_protected.sum().round(1)
# richness_chart = bar_chart(df, column, 'mean_richness')
# rsr_chart = bar_chart(df, column, 'mean_rsr')
# carbon_lost = bar_chart(df, column, 'carbon_lost')
# crop_expansion = bar_chart(df, column, 'crop_expansion')
# human_impact = bar_chart(df, column, 'human_impact')
with stats_col:
with st.container():
f"{total_percent}% Continental US Covered"
st.altair_chart(area_plot(df, column), use_container_width=True)
with st.container():
if show_richness:
"Species Richness"
st.altair_chart(richness_chart, use_container_width=True)
if show_rsr:
"Range-Size Rarity"
st.altair_chart(rsr_chart, use_container_width=True)
if show_carbon_lost:
"Carbon Lost ('02-'22)"
st.altair_chart(carbon_loss_chart, use_container_width=True)
if show_crop_expansion:
"Crop Expansion"
st.altair_chart(crop_expansion_chart, use_container_width=True)
if show_human_impact:
"Human Impact"
st.altair_chart(hi_chart, use_container_width=True)
if show_irr_carbon:
"Irrecoverable Carbon"
st.altair_chart(irr_carbon_chart, use_container_width=True)
if show_man_carbon:
"Manageable Carbon"
st.altair_chart(man_carbon_chart, use_container_width=True)
if show_bio_loss:
"Biodiversity Intactness Loss"
st.altair_chart(bio_intact_loss_chart, use_container_width=True)
if show_forest_loss:
"Forest Integrity Loss"
st.altair_chart(forest_integrity_loss_chart, use_container_width=True)
# charts displayed based on color_by variable
# +
st.divider()
footer = st.container()
with footer:
'''
## Custom queries
Input custom python code below to interactively explore the data.
'''
col2_1, col2_2 = st.columns(2)
with col2_1:
query = st.text_area(
label = "Python code:",
value = sample_q,
height = 300)
with col2_2:
"Output table:"
df = eval(query)
st.write(df.to_pandas())
st.divider()
'''
## Credits
Author: Cassie Buhler & Carl Boettiger, UC Berkeley
License: BSD-2-clause
### Data sources
- US Protected Areas Database v3 by USGS. Data: https://beta.source.coop/cboettig/us-pad-3. Citation: https://doi.org/10.5066/P9Q9LQ4B, License: Public Domain
- Imperiled Species Richness and Range-Size-Rarity from NatureServe (2022). Data: https://beta.source.coop/repositories/cboettig/mobi. License CC-BY-NC-ND
- Carbon-loss and farming impact by Vizzuality, on https://beta.source.coop/repositories/vizzuality/lg-land-carbon-data. Citation: https://doi.org/10.1101/2023.11.01.565036, License: CC-BY
- Human Footprint by Vizzuality, on https://beta.source.coop/repositories/vizzuality/hfp-100. Citation: https://doi.org/10.3389/frsen.2023.1130896, License: Public Domain
- Fire polygons by USGS, reprocessed to PMTiles on https://beta.source.coop/cboettig/fire/. License: Public Domain
- Irrecoverable Carbon from Conservation International, reprocessed to COG on https://beta.source.coop/cboettig/carbon, citation: https://doi.org/10.1038/s41893-021-00803-6, License: CC-BY-NC
- Climate and Economic Justice Screening Tool, US Council on Environmental Quality, Justice40, data: https://beta.source.coop/repositories/cboettig/justice40/description/, License: Public Domain
### Software
Proudly built with a free and Open Source software stack: Streamlit (reactive application), HuggingFace (application hosting), Source.Coop (data hosting),
using cloud-native data serializations in COG, PMTiles, and GeoParquet. Coded in pure python using leafmap and duckdb. Map styling with [MapLibre](https://maplibre.org/).
'''